From e441067f93e44c2a650b92daa3ff51a86154294d Mon Sep 17 00:00:00 2001
From: thorpej <thorpej@NetBSD.org>
Date: Sat, 21 Mar 2020 16:56:00 +0000
Subject: [PATCH] Use the 64-bit DMA tag, if available, with some constraints:
 - The data buffers can come from anywhere in the 64-bit region, but   the
 upper 16 bits of the data buffer address is stored in a single   register, so
 just treat that as 0 and create a 48-bit restricted   DMA tag. - The
 descriptor address registers share a single register for the   upper 32-bits,
 so enforce a 4G boundary when allocating memory   for the descriptors.

---
 sys/dev/pci/if_vge.c | 47 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/sys/dev/pci/if_vge.c b/sys/dev/pci/if_vge.c
index f5b0f976cd8d..a3f88cb7f127 100644
--- a/sys/dev/pci/if_vge.c
+++ b/sys/dev/pci/if_vge.c
@@ -1,4 +1,4 @@
-/* $NetBSD: if_vge.c,v 1.79 2020/01/30 05:24:53 thorpej Exp $ */
+/* $NetBSD: if_vge.c,v 1.80 2020/03/21 16:56:00 thorpej Exp $ */
 
 /*-
  * Copyright (c) 2004
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_vge.c,v 1.79 2020/01/30 05:24:53 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_vge.c,v 1.80 2020/03/21 16:56:00 thorpej Exp $");
 
 /*
  * VIA Networking Technologies VT612x PCI gigabit ethernet NIC driver.
@@ -75,11 +75,14 @@ __KERNEL_RCSID(0, "$NetBSD: if_vge.c,v 1.79 2020/01/30 05:24:53 thorpej Exp $");
  * The other issue has to do with the way 64-bit addresses are handled.
  * The DMA descriptors only allow you to specify 48 bits of addressing
  * information. The remaining 16 bits are specified using one of the
- * I/O registers. If you only have a 32-bit system, then this isn't
- * an issue, but if you have a 64-bit system and more than 4GB of
- * memory, you must have to make sure your network data buffers reside
+ * I/O registers (VGE_DATABUF_HIADDR). If you only have a 32-bit system,
+ * then this isn't an issue, but if you have a 64-bit system and more than
+ * 4GB of memory, you must have to make sure your network data buffers reside
  * in the same 48-bit 'segment.'
  *
+ * Furthermore, the descriptors must also all reside within the same 32-bit
+ * 'segment' (see VGE_TXDESC_HIADDR).
+ *
  * Special thanks to Ryan Fu at VIA Networking for providing documentation
  * and sample NICs for testing.
  */
@@ -128,8 +131,8 @@ __KERNEL_RCSID(0, "$NetBSD: if_vge.c,v 1.79 2020/01/30 05:24:53 thorpej Exp $");
 #define VGE_NEXT_RXDESC(x)	((x + 1) & VGE_NRXDESC_MASK)
 #define VGE_PREV_RXDESC(x)	((x - 1) & VGE_NRXDESC_MASK)
 
-#define VGE_ADDR_LO(y)		((uint64_t)(y) & 0xFFFFFFFF)
-#define VGE_ADDR_HI(y)		((uint64_t)(y) >> 32)
+#define VGE_ADDR_LO(y)		BUS_ADDR_LO32(y)
+#define VGE_ADDR_HI(y)		BUS_ADDR_HI32(y)
 #define VGE_BUFLEN(y)		((y) & 0x7FFF)
 #define ETHER_PAD_LEN		(ETHER_MIN_LEN - ETHER_CRC_LEN)
 
@@ -781,10 +784,17 @@ vge_allocmem(struct vge_softc *sc)
 
 	/*
 	 * Allocate memory for control data.
+	 *
+	 * NOTE: This must all fit within the same 4GB segment.  The
+	 * "boundary" argument to bus_dmamem_alloc() will end up as
+	 * 4GB on 64-bit platforms and 0 ("no boundary constraint") on
+	 * 32-bit platformds.
 	 */
 
 	error = bus_dmamem_alloc(sc->sc_dmat, sizeof(struct vge_control_data),
-	     VGE_RING_ALIGN, 0, &seg, 1, &nseg, BUS_DMA_NOWAIT);
+	     VGE_RING_ALIGN,
+	     (bus_size_t)(1ULL << 32),
+	     &seg, 1, &nseg, BUS_DMA_NOWAIT);
 	if (error) {
 		aprint_error_dev(sc->sc_dev,
 		    "could not allocate control data dma memory\n");
@@ -958,10 +968,24 @@ vge_attach(device_t parent, device_t self, void *aux)
 	vge_clrwol(sc);
 
 	/*
-	 * Use the 32bit tag. Hardware supports 48bit physical addresses,
-	 * but we don't use that for now.
+	 * The hardware supports 64-bit DMA addresses, but it's a little
+	 * complicated (see large comment about the hardware near the top
+	 * of the file).  TL;DR -- restrict ourselves to 48-bit.
 	 */
-	sc->sc_dmat = pa->pa_dmat;
+	if (pci_dma64_available(pa)) {
+		if (bus_dmatag_subregion(pa->pa_dmat64,
+					 0,
+					 (bus_addr_t)(1ULL << 48),
+					 &sc->sc_dmat,
+					 BUS_DMA_WAITOK) != 0) {
+			aprint_error_dev(self,
+			    "WARNING: failed to restrict dma range,"
+			    " falling back to parent bus dma range\n");
+			sc->sc_dmat = pa->pa_dmat64;
+		}
+	} else {
+		sc->sc_dmat = pa->pa_dmat;
+	}
 
 	if (vge_allocmem(sc) != 0)
 		return;
@@ -1793,6 +1817,7 @@ vge_init(struct ifnet *ifp)
 	 * Note that we only use one transmit queue.
 	 */
 
+	CSR_WRITE_4(sc, VGE_TXDESC_HIADDR, VGE_ADDR_HI(VGE_CDTXADDR(sc, 0)));
 	CSR_WRITE_4(sc, VGE_TXDESC_ADDR_LO0, VGE_ADDR_LO(VGE_CDTXADDR(sc, 0)));
 	CSR_WRITE_2(sc, VGE_TXDESCNUM, VGE_NTXDESC - 1);