diff --git a/sys/arch/aarch64/aarch64/bus_space.c b/sys/arch/aarch64/aarch64/bus_space.c index 2e2951d9cf09..1b929af5feed 100644 --- a/sys/arch/aarch64/aarch64/bus_space.c +++ b/sys/arch/aarch64/aarch64/bus_space.c @@ -1,4 +1,4 @@ -/* $NetBSD: bus_space.c,v 1.8 2019/01/27 02:08:36 pgoyette Exp $ */ +/* $NetBSD: bus_space.c,v 1.9 2019/12/28 17:19:43 jmcneill Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu @@ -27,7 +27,7 @@ */ #include -__KERNEL_RCSID(1, "$NetBSD: bus_space.c,v 1.8 2019/01/27 02:08:36 pgoyette Exp $"); +__KERNEL_RCSID(1, "$NetBSD: bus_space.c,v 1.9 2019/12/28 17:19:43 jmcneill Exp $"); #include #include @@ -561,6 +561,8 @@ generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flag, pmapflags = PMAP_WRITE_COMBINE; else if ((flag & BUS_SPACE_MAP_CACHEABLE) != 0) pmapflags = PMAP_WRITE_BACK; + else if ((flag & _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED) != 0) + pmapflags = PMAP_DEV_SO; else pmapflags = PMAP_DEV; diff --git a/sys/arch/aarch64/aarch64/genassym.cf b/sys/arch/aarch64/aarch64/genassym.cf index 4333e34bb6f5..7f63c54f0b47 100644 --- a/sys/arch/aarch64/aarch64/genassym.cf +++ b/sys/arch/aarch64/aarch64/genassym.cf @@ -1,4 +1,4 @@ -# $NetBSD: genassym.cf,v 1.16 2019/12/27 18:56:47 jmcneill Exp $ +# $NetBSD: genassym.cf,v 1.17 2019/12/28 17:19:43 jmcneill Exp $ #- # Copyright (c) 2014 The NetBSD Foundation, Inc. # All rights reserved. @@ -335,6 +335,7 @@ define MAIR_ATTR0 MAIR_ATTR0 define MAIR_ATTR1 MAIR_ATTR1 define MAIR_ATTR2 MAIR_ATTR2 define MAIR_ATTR3 MAIR_ATTR3 +define MAIR_ATTR4 MAIR_ATTR4 define MAIR_DEVICE_nGnRnE MAIR_DEVICE_nGnRnE define MAIR_DEVICE_nGnRE MAIR_DEVICE_nGnRE define MAIR_NORMAL_NC MAIR_NORMAL_NC diff --git a/sys/arch/aarch64/aarch64/locore.S b/sys/arch/aarch64/aarch64/locore.S index c50de7854eea..76d494fbf20c 100644 --- a/sys/arch/aarch64/aarch64/locore.S +++ b/sys/arch/aarch64/aarch64/locore.S @@ -1,4 +1,4 @@ -/* $NetBSD: locore.S,v 1.48 2019/12/27 18:56:47 jmcneill Exp $ */ +/* $NetBSD: locore.S,v 1.49 2019/12/28 17:19:43 jmcneill Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu @@ -38,13 +38,14 @@ #include #include "assym.h" -RCSID("$NetBSD: locore.S,v 1.48 2019/12/27 18:56:47 jmcneill Exp $") +RCSID("$NetBSD: locore.S,v 1.49 2019/12/28 17:19:43 jmcneill Exp $") -#ifdef AARCH64_DEVICE_MEM_STRICTLY_ORDERED +#ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRnE #else #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRE #endif +#define MAIR_DEVICE_MEM_SO MAIR_DEVICE_nGnRnE /*#define DEBUG_LOCORE /* debug print */ /*#define DEBUG_LOCORE_PRINT_LOCK /* avoid mixing AP's output */ @@ -953,7 +954,8 @@ mair_setting: __SHIFTIN(MAIR_NORMAL_WB, MAIR_ATTR0) | \ __SHIFTIN(MAIR_NORMAL_NC, MAIR_ATTR1) | \ __SHIFTIN(MAIR_NORMAL_WT, MAIR_ATTR2) | \ - __SHIFTIN(MAIR_DEVICE_MEM, MAIR_ATTR3)) + __SHIFTIN(MAIR_DEVICE_MEM, MAIR_ATTR3) | \ + __SHIFTIN(MAIR_DEVICE_MEM_SO, MAIR_ATTR4)) #define VIRT_BIT 48 diff --git a/sys/arch/aarch64/aarch64/pmap.c b/sys/arch/aarch64/aarch64/pmap.c index 75c24356e738..7548ce91107d 100644 --- a/sys/arch/aarch64/aarch64/pmap.c +++ b/sys/arch/aarch64/aarch64/pmap.c @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.c,v 1.57 2019/12/27 18:56:47 jmcneill Exp $ */ +/* $NetBSD: pmap.c,v 1.58 2019/12/28 17:19:43 jmcneill Exp $ */ /* * Copyright (c) 2017 Ryo Shimizu @@ -27,7 +27,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.57 2019/12/27 18:56:47 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.58 2019/12/28 17:19:43 jmcneill Exp $"); #include "opt_arm_debug.h" #include "opt_ddb.h" @@ -951,7 +951,10 @@ _pmap_pte_adjust_cacheflags(pt_entry_t pte, u_int flags) pte &= ~LX_BLKPAG_ATTR_MASK; - switch (flags & (PMAP_CACHE_MASK|PMAP_DEV)) { + switch (flags & (PMAP_CACHE_MASK|PMAP_DEV_MASK)) { + case PMAP_DEV_SO ... PMAP_DEV_SO | PMAP_CACHE_MASK: + pte |= LX_BLKPAG_ATTR_DEVICE_MEM_SO; /* Device-nGnRnE */ + break; case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK: pte |= LX_BLKPAG_ATTR_DEVICE_MEM; /* Device-nGnRE */ break; diff --git a/sys/arch/aarch64/conf/files.aarch64 b/sys/arch/aarch64/conf/files.aarch64 index fefba0c3c476..70365b6f1e86 100644 --- a/sys/arch/aarch64/conf/files.aarch64 +++ b/sys/arch/aarch64/conf/files.aarch64 @@ -1,10 +1,10 @@ -# $NetBSD: files.aarch64,v 1.15 2019/12/27 18:56:47 jmcneill Exp $ +# $NetBSD: files.aarch64,v 1.16 2019/12/28 17:19:43 jmcneill Exp $ defflag opt_cpuoptions.h AARCH64_ALIGNMENT_CHECK defflag opt_cpuoptions.h AARCH64_EL0_STACK_ALIGNMENT_CHECK defflag opt_cpuoptions.h AARCH64_EL1_STACK_ALIGNMENT_CHECK defflag opt_cpuoptions.h AARCH64_HAVE_L2CTLR -defflag opt_cpuoptions.h AARCH64_DEVICE_MEM_STRICTLY_ORDERED +defflag opt_cpuoptions.h AARCH64_DEVICE_MEM_STRONGLY_ORDERED defflag opt_cputypes.h CPU_ARMV8 defflag opt_cputypes.h CPU_CORTEX: CPU_ARMV8 diff --git a/sys/arch/aarch64/include/pmap.h b/sys/arch/aarch64/include/pmap.h index 95ba0ac8c797..af972991834d 100644 --- a/sys/arch/aarch64/include/pmap.h +++ b/sys/arch/aarch64/include/pmap.h @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.h,v 1.27 2019/12/27 18:56:47 jmcneill Exp $ */ +/* $NetBSD: pmap.h,v 1.28 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 2014 The NetBSD Foundation, Inc. @@ -114,6 +114,7 @@ struct vm_page_md { #define LX_BLKPAG_ATTR_NORMAL_NC __SHIFTIN(1, LX_BLKPAG_ATTR_INDX) #define LX_BLKPAG_ATTR_NORMAL_WT __SHIFTIN(2, LX_BLKPAG_ATTR_INDX) #define LX_BLKPAG_ATTR_DEVICE_MEM __SHIFTIN(3, LX_BLKPAG_ATTR_INDX) +#define LX_BLKPAG_ATTR_DEVICE_MEM_SO __SHIFTIN(4, LX_BLKPAG_ATTR_INDX) #define LX_BLKPAG_ATTR_MASK LX_BLKPAG_ATTR_INDX #define lxpde_pa(pde) ((paddr_t)((pde) & LX_TBL_PA)) @@ -228,6 +229,8 @@ paddr_t pmap_alloc_pdp(struct pmap *, struct vm_page **, int, bool); #define PMAP_PTE 0x10000000 /* kenter_pa */ #define PMAP_DEV 0x20000000 /* kenter_pa */ +#define PMAP_DEV_SO 0x40000000 /* kenter_pa */ +#define PMAP_DEV_MASK (PMAP_DEV | PMAP_DEV_SO) static inline u_int aarch64_mmap_flags(paddr_t mdpgno) @@ -235,12 +238,13 @@ aarch64_mmap_flags(paddr_t mdpgno) u_int nflag, pflag; /* - * aarch64 arch has 4 memory attribute: + * aarch64 arch has 5 memory attribute: * * WriteBack - write back cache * WriteThru - wite through cache * NoCache - no cache * Device(nGnRE) - no Gathering, no Reordering, Early write ack + * Device(nGnRnE) - no Gathering, no Reordering, no Early write ack * * but pmap has PMAP_{NOCACHE,WRITE_COMBINE,WRITE_BACK} flags. */ diff --git a/sys/arch/arm/acpi/acpi_machdep.c b/sys/arch/arm/acpi/acpi_machdep.c index 6489be15a462..9cd7e49b4bc3 100644 --- a/sys/arch/arm/acpi/acpi_machdep.c +++ b/sys/arch/arm/acpi/acpi_machdep.c @@ -1,4 +1,4 @@ -/* $NetBSD: acpi_machdep.c,v 1.12 2019/12/23 15:34:23 jmcneill Exp $ */ +/* $NetBSD: acpi_machdep.c,v 1.13 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -32,7 +32,7 @@ #include "pci.h" #include -__KERNEL_RCSID(0, "$NetBSD: acpi_machdep.c,v 1.12 2019/12/23 15:34:23 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: acpi_machdep.c,v 1.13 2019/12/28 17:19:43 jmcneill Exp $"); #include #include @@ -325,13 +325,27 @@ acpi_md_gtdt_probe(ACPI_GTDT_HEADER *hdrp, void *aux) return AE_OK; } +#if NPCI > 0 +static struct bus_space acpi_md_mcfg_bs_tag; + +static int +acpi_md_mcfg_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flag, + bus_space_handle_t *bshp) +{ + return arm_generic_bs_tag.bs_map(t, bpa, size, + flag | _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED, bshp); +} +#endif + void acpi_md_callback(struct acpi_softc *sc) { ACPI_TABLE_HEADER *hdrp; #if NPCI > 0 - acpimcfg_init(&arm_generic_bs_tag, NULL); + acpi_md_mcfg_bs_tag = arm_generic_bs_tag; + acpi_md_mcfg_bs_tag.bs_map = acpi_md_mcfg_bs_map; + acpimcfg_init(&acpi_md_mcfg_bs_tag, NULL); #endif if (acpi_madt_map() != AE_OK) diff --git a/sys/arch/arm/acpi/acpipchb.c b/sys/arch/arm/acpi/acpipchb.c index f989648e2520..228f42fc1738 100644 --- a/sys/arch/arm/acpi/acpipchb.c +++ b/sys/arch/arm/acpi/acpipchb.c @@ -1,4 +1,4 @@ -/* $NetBSD: acpipchb.c,v 1.13 2019/10/15 13:27:50 jmcneill Exp $ */ +/* $NetBSD: acpipchb.c,v 1.14 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 2018 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: acpipchb.c,v 1.13 2019/10/15 13:27:50 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: acpipchb.c,v 1.14 2019/12/28 17:19:43 jmcneill Exp $"); #include #include @@ -75,6 +75,8 @@ struct acpipchb_bus_space { int (*map)(void *, bus_addr_t, bus_size_t, int, bus_space_handle_t *); + + int flags; }; struct acpipchb_softc { @@ -157,7 +159,8 @@ acpipchb_amazon_graviton_map(ACPI_HANDLE handle, UINT32 level, void *ctx, void * return AE_NOT_FOUND; } - error = bus_space_map(ap->ap_bst, mem->ar_base, mem->ar_length, 0, &ap->ap_conf_bsh); + error = bus_space_map(ap->ap_bst, mem->ar_base, mem->ar_length, + _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED, &ap->ap_conf_bsh); if (error != 0) return AE_NO_MEMORY; @@ -309,6 +312,11 @@ acpipchb_bus_space_map(void *t, bus_addr_t bpa, bus_size_t size, int flag, if (size == 0) return ERANGE; + if ((abs->flags & PCI_FLAGS_IO_OKAY) != 0) { + /* Force strongly ordered mapping for all I/O space */ + flag = _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED; + } + for (i = 0; i < abs->nrange; i++) { struct acpipchb_bus_range * const range = &abs->range[i]; if (bpa >= range->min && bpa + size - 1 <= range->max) @@ -377,6 +385,7 @@ acpipchb_setup_ranges_cb(ACPI_RESOURCE *res, void *ctx) abs->bs = *sc->sc_memt; abs->bs.bs_cookie = abs; abs->map = abs->bs.bs_map; + abs->flags = pci_flags; abs->bs.bs_map = acpipchb_bus_space_map; if ((pci_flags & PCI_FLAGS_IO_OKAY) != 0) pba->pba_iot = &abs->bs; diff --git a/sys/arch/arm/fdt/pcihost_fdt.c b/sys/arch/arm/fdt/pcihost_fdt.c index e7bce54baa5d..2129776252f6 100644 --- a/sys/arch/arm/fdt/pcihost_fdt.c +++ b/sys/arch/arm/fdt/pcihost_fdt.c @@ -1,4 +1,4 @@ -/* $NetBSD: pcihost_fdt.c,v 1.11 2019/06/23 22:06:03 jmcneill Exp $ */ +/* $NetBSD: pcihost_fdt.c,v 1.12 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 2018 Jared D. McNeill @@ -27,7 +27,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: pcihost_fdt.c,v 1.11 2019/06/23 22:06:03 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pcihost_fdt.c,v 1.12 2019/12/28 17:19:43 jmcneill Exp $"); #include #include @@ -126,7 +126,8 @@ pcihost_attach(device_t parent, device_t self, void *aux) sc->sc_dmat = faa->faa_dmat; sc->sc_bst = faa->faa_bst; sc->sc_phandle = faa->faa_phandle; - error = bus_space_map(sc->sc_bst, cs_addr, cs_size, 0, &sc->sc_bsh); + error = bus_space_map(sc->sc_bst, cs_addr, cs_size, + _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED, &sc->sc_bsh); if (error) { aprint_error(": couldn't map registers: %d\n", error); return; @@ -231,12 +232,14 @@ pcihost_config(struct pcihost_softc *sc) pibs->bst = *sc->sc_bst; pibs->bst.bs_cookie = pibs; pibs->map = pibs->bst.bs_map; + pibs->flags = PCI_FLAGS_IO_OKAY; pibs->bst.bs_map = pcihost_bus_space_map; struct pcih_bus_space * const pmbs = &sc->sc_mem; pmbs->bst = *sc->sc_bst; pmbs->bst.bs_cookie = pmbs; pmbs->map = pmbs->bst.bs_map; + pmbs->flags = PCI_FLAGS_MEM_OKAY; pmbs->bst.bs_map = pcihost_bus_space_map; /* @@ -634,6 +637,11 @@ pcihost_bus_space_map(void *t, bus_addr_t bpa, bus_size_t size, int flag, { struct pcih_bus_space * const pbs = t; + if ((pbs->flags & PCI_FLAGS_IO_OKAY) != 0) { + /* Force strongly ordered mapping for all I/O space */ + flag = _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED; + } + for (size_t i = 0; i < pbs->nranges; i++) { const bus_addr_t rmin = pbs->ranges[i].bpci; const bus_addr_t rmax = pbs->ranges[i].bpci - 1 + pbs->ranges[i].size; diff --git a/sys/arch/arm/fdt/pcihost_fdtvar.h b/sys/arch/arm/fdt/pcihost_fdtvar.h index 1fe08babd64a..4006d7903416 100644 --- a/sys/arch/arm/fdt/pcihost_fdtvar.h +++ b/sys/arch/arm/fdt/pcihost_fdtvar.h @@ -1,4 +1,4 @@ -/* $NetBSD: pcihost_fdtvar.h,v 1.2 2019/06/12 10:13:44 jmcneill Exp $ */ +/* $NetBSD: pcihost_fdtvar.h,v 1.3 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 2018 Jared D. McNeill @@ -52,6 +52,8 @@ struct pcih_bus_space { int (*map)(void *, bus_addr_t, bus_size_t, int, bus_space_handle_t *); + int flags; + struct space_range { bus_addr_t bpci; bus_addr_t bbus; diff --git a/sys/arch/arm/include/bus_defs.h b/sys/arch/arm/include/bus_defs.h index 80b0246718ce..d9f4c9c7ee6e 100644 --- a/sys/arch/arm/include/bus_defs.h +++ b/sys/arch/arm/include/bus_defs.h @@ -1,4 +1,4 @@ -/* $NetBSD: bus_defs.h,v 1.13 2018/11/19 10:45:47 jmcneill Exp $ */ +/* $NetBSD: bus_defs.h,v 1.14 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. @@ -97,6 +97,13 @@ typedef u_long bus_space_handle_t; #define BUS_SPACE_MAP_LINEAR 0x02 #define BUS_SPACE_MAP_PREFETCHABLE 0x04 +#define BUS_SPACE_MAP_BUS1 0x0100 +#define BUS_SPACE_MAP_BUS2 0x0200 +#define BUS_SPACE_MAP_BUS3 0x0400 +#define BUS_SPACE_MAP_BUS4 0x0800 + +#define _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED BUS_SPACE_MAP_BUS1 + struct bus_space { /* cookie */ void *bs_cookie; diff --git a/sys/arch/arm/nvidia/tegra_pcie.c b/sys/arch/arm/nvidia/tegra_pcie.c index fa2b26467b05..924b3afdb89f 100644 --- a/sys/arch/arm/nvidia/tegra_pcie.c +++ b/sys/arch/arm/nvidia/tegra_pcie.c @@ -1,4 +1,4 @@ -/* $NetBSD: tegra_pcie.c,v 1.26 2019/03/12 18:46:20 jakllsch Exp $ */ +/* $NetBSD: tegra_pcie.c,v 1.27 2019/12/28 17:19:43 jmcneill Exp $ */ /*- * Copyright (c) 2015 Jared D. McNeill @@ -27,7 +27,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: tegra_pcie.c,v 1.26 2019/03/12 18:46:20 jakllsch Exp $"); +__KERNEL_RCSID(0, "$NetBSD: tegra_pcie.c,v 1.27 2019/12/28 17:19:43 jmcneill Exp $"); #include #include @@ -188,8 +188,8 @@ tegra_pcie_attach(device_t parent, device_t self, void *aux) aprint_error(": couldn't map pads registers: %d\n", error); return; } - error = bus_space_map(sc->sc_bst, cs_addr, cs_size, 0, - &sc->sc_bsh_rpconf); + error = bus_space_map(sc->sc_bst, cs_addr, cs_size, + _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED, &sc->sc_bsh_rpconf); if (error) { aprint_error(": couldn't map cs registers: %d\n", error); return; @@ -565,7 +565,8 @@ tegra_pcie_conf_frag_map(struct tegra_pcie_softc * const sc, uint bus, } a = TEGRA_PCIE_EXTC_BASE + (bus << 16) + (frg << 24); - if (bus_space_map(sc->sc_bst, a, 1 << 16, 0, + if (bus_space_map(sc->sc_bst, a, 1 << 16, + _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED, &sc->sc_bsh_extc[bus-1][frg]) != 0) device_printf(sc->sc_dev, "couldn't map PCIE " "configuration for bus %u fragment %#x", bus, frg); diff --git a/sys/arch/arm/rockchip/rk3399_pcie.c b/sys/arch/arm/rockchip/rk3399_pcie.c index 919bd7df932a..d6822e87c107 100644 --- a/sys/arch/arm/rockchip/rk3399_pcie.c +++ b/sys/arch/arm/rockchip/rk3399_pcie.c @@ -1,4 +1,4 @@ -/* $NetBSD: rk3399_pcie.c,v 1.8 2019/12/07 16:00:36 jmcneill Exp $ */ +/* $NetBSD: rk3399_pcie.c,v 1.9 2019/12/28 17:19:44 jmcneill Exp $ */ /* * Copyright (c) 2018 Mark Kettenis * @@ -17,7 +17,7 @@ #include -__KERNEL_RCSID(1, "$NetBSD: rk3399_pcie.c,v 1.8 2019/12/07 16:00:36 jmcneill Exp $"); +__KERNEL_RCSID(1, "$NetBSD: rk3399_pcie.c,v 1.9 2019/12/28 17:19:44 jmcneill Exp $"); #include #include @@ -228,8 +228,9 @@ rkpcie_attach(device_t parent, device_t self, void *aux) return; } - if (bus_space_map(sc->sc_iot, sc->sc_apb_addr, sc->sc_apb_size, 0, &sc->sc_ioh) != 0 || - bus_space_map(sc->sc_iot, sc->sc_axi_addr, sc->sc_axi_size, 0, &sc->sc_axi_ioh) != 0) { + const int mapflags = _ARM_BUS_SPACE_MAP_STRONGLY_ORDERED; + if (bus_space_map(sc->sc_iot, sc->sc_apb_addr, sc->sc_apb_size, mapflags, &sc->sc_ioh) != 0 || + bus_space_map(sc->sc_iot, sc->sc_axi_addr, sc->sc_axi_size, mapflags, &sc->sc_axi_ioh) != 0) { printf(": can't map registers\n"); sc->sc_axi_size = 0; sc->sc_apb_size = 0;