diff --git a/sys/arch/arm/cortex/files.cortex b/sys/arch/arm/cortex/files.cortex index 6e113c781aef..558731d20d23 100644 --- a/sys/arch/arm/cortex/files.cortex +++ b/sys/arch/arm/cortex/files.cortex @@ -1,4 +1,4 @@ -# $NetBSD: files.cortex,v 1.9 2018/10/21 00:42:05 jmcneill Exp $ +# $NetBSD: files.cortex,v 1.10 2018/11/09 23:36:24 jmcneill Exp $ defflag opt_cpu_in_cksum.h NEON_IN_CKSUM @@ -20,6 +20,7 @@ file arch/arm/cortex/gic_v2m.c armgic & pci & __have_pci_msi_msix # ARM Generic Interrupt Controller v3+ device gicvthree: pic, pic_splfuncs file arch/arm/cortex/gicv3.c gicvthree +file arch/arm/cortex/gicv3_its.c gicvthree # ARM PL310 L2 Cache Controller(initially on Cortex-A9) device arml2cc diff --git a/sys/arch/arm/cortex/gic_reg.h b/sys/arch/arm/cortex/gic_reg.h index a2c75146bd0e..efb8b9393c52 100644 --- a/sys/arch/arm/cortex/gic_reg.h +++ b/sys/arch/arm/cortex/gic_reg.h @@ -1,4 +1,4 @@ -/* $NetBSD: gic_reg.h,v 1.7 2018/08/08 19:01:54 jmcneill Exp $ */ +/* $NetBSD: gic_reg.h,v 1.8 2018/11/09 23:36:24 jmcneill Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. * All rights reserved. @@ -149,6 +149,12 @@ #define GICD_CTRL_EnableGrp1NS __BIT(1) // GICv3 #define GICD_CTRL_Enable __BIT(0) +#define GICD_TYPER_No1N __BIT(25) // GICv3 +#define GICD_TYPER_A3V __BIT(24) // GICv3 +#define GICD_TYPER_IDbits __BITS(23,19) // GICv3 +#define GICD_TYPER_DVIS __BIT(18) // GICv3 +#define GICD_TYPER_LPIS __BIT(17) // GICv3 +#define GICD_TYPER_MBIS __BIT(16) // GICv3 #define GICD_TYPER_LSPI __BITS(15,11) #define GICD_TYPER_SecurityExtn __BIT(10) #define GICD_TYPER_CPUNumber __BITS(7,5) @@ -190,7 +196,7 @@ #define GICD_IROUTER_Aff1 __BITS(15,8) #define GICD_IROUTER_Aff0 __BITS(7,0) -#define GICR_CTRL 0x0000 // Redistributor Control Register +#define GICR_CTLR 0x0000 // Redistributor Control Register #define GICR_IIDR 0x0004 // Implementor Identification Register #define GICR_TYPER 0x0008 // Redistributor Type Register #define GICR_STATUSR 0x0010 // Error Reporting Status Register, optional @@ -215,12 +221,12 @@ #define GICR_IGRPMODR0 0x10D00 // Interrupt Group Modifier Register 0 #define GICR_NSACR 0x10E00 // Non-Secure Access Control Register -#define GICR_CTRL_UWP __BIT(31) -#define GICR_CTRL_DPG1S __BIT(26) -#define GICR_CTRL_DPG1NS __BIT(25) -#define GICR_CTRL_DPG0 __BIT(24) -#define GICR_CTRL_RWP __BIT(3) -#define GICR_CTRL_Enable_LPIs __BIT(0) +#define GICR_CTLR_UWP __BIT(31) +#define GICR_CTLR_DPG1S __BIT(26) +#define GICR_CTLR_DPG1NS __BIT(25) +#define GICR_CTLR_DPG0 __BIT(24) +#define GICR_CTLR_RWP __BIT(3) +#define GICR_CTLR_Enable_LPIs __BIT(0) #define GICR_TYPER_Affinity_Value __BITS(63,32) #define GICR_TYPER_Affinity_Value_Aff3 __BITS(63,56) @@ -238,6 +244,40 @@ #define GICR_WAKER_ChildrenAsleep __BIT(2) #define GICR_WAKER_ProcessorSleep __BIT(1) +#define GICR_PROPBASER_OuterCache __BITS(58,56) +#define GICR_PROPBASER_Physical_Address __BITS(51,12) +#define GICR_PROPBASER_Shareability __BITS(11,10) +#define GICR_PROPBASER_InnerCache __BITS(9,7) +#define GICR_PROPBASER_IDbits __BITS(4,0) + +#define GICR_PENDBASER_PTZ __BIT(62) +#define GICR_PENDBASER_OuterCache __BITS(58,56) +#define GICR_PENDBASER_Physical_Address __BITS(51,16) +#define GICR_PENDBASER_Shareability __BITS(11,10) +#define GICR_PENDBASER_InnerCache __BITS(9,7) + +#define GICR_Shareability_NS 0 // Non-shareable +#define GICR_Shareability_IS 1 // Inner Shareable +#define GICR_Shareability_OS 2 // Outer Shareable + +#define GICR_Cache_DEVICE_nGnRnE 0 // Device-nGnRnE +#define GICR_Cache_NORMAL_NC 1 // Non-cacheable +#define GICR_Cache_NORMAL_RA_WT 2 // Cacheable Read-allocate, Write-through +#define GICR_Cache_NORMAL_RA_WB 3 // Cacheable Read-allocate, Write-back +#define GICR_Cache_NORMAL_WA_WT 4 // Cacheable Write-allocate, Write-through +#define GICR_Cache_NORMAL_WA_WB 5 // Cacheable Write-allocate, Write-back +#define GICR_Cache_NORMAL_RA_WA_WT 6 // Cacheable Read-allocate, Write-allocate, Write-through +#define GICR_Cache_NORMAL_RA_WA_WB 7 // Cacheable Read-allocate, Write-allocate, Write-back + +/* + * GICv3 Locality-specific Peripheral Interrupts + */ + +#define GIC_LPI_BASE 0x2000 // Base LPI INTID + +#define GIC_LPICONF_Priority __BITS(7,2) +#define GIC_LPICONF_Res1 __BIT(1) +#define GIC_LPICONF_Enable __BIT(0) /* * GICv1 names @@ -266,7 +306,9 @@ #define GICv1_ICCHPIR GICC_HPPIR #define GICv1_ICCIIDR GICC_IIDR -/* GICv2m (MSI) */ +/* + * GICv2m (MSI) + */ #define GIC_MSI_TYPER 0x0008 #define GIC_MSI_SETSPI 0x0040 @@ -276,4 +318,106 @@ #define GIC_MSI_TYPER_BASE __BITS(25,16) // Starting SPI of MSIs #define GIC_MSI_TYPER_NUMBER __BITS(9,0) // Count of MSIs +/* + * GICv3 Interrupt Translation Service (ITS) + */ + +#define GITS_CTLR 0x00000 // ITS control register +#define GITS_IIDR 0x00004 // ITS Identification register +#define GITS_TYPER 0x00008 // ITS Type register +#define GITS_CBASER 0x00080 // ITS Command Queue Descriptor +#define GITS_CWRITER 0x00088 // ITS Write register +#define GITS_CREADR 0x00090 // ITS Read register +#define GITS_BASERn(n) (0x00100+8*(n)) // ITS Translation Table Descriptors +#define GITS_PIDR2 0x0FFE8 // ITS Peripheral ID2 Register +#define GITS_TRANSLATER 0x10040 // ITS Translation register + +#define GITS_CTLR_Quiescent __BIT(31) +#define GITS_CTLR_ITS_Number __BITS(7,4) +#define GITS_CTLR_ImDe __BIT(1) +#define GITS_CTLR_Enabled __BIT(0) + +#define GITS_TYPER_VMOVP __BIT(37) +#define GITS_TYPER_CIL __BIT(36) +#define GITS_TYPER_CIDbits __BITS(35,32) +#define GITS_TYPER_HCC __BITS(31,24) +#define GITS_TYPER_PTA __BIT(19) +#define GITS_TYPER_SEIS __BIT(18) +#define GITS_TYPER_Devbits __BITS(17,13) +#define GITS_TYPER_ID_bits __BITS(12,8) +#define GITS_TYPER_ITT_entry_size __BITS(7,4) +#define GITS_TYPER_CCT __BIT(2) +#define GITS_TYPER_Virtual __BIT(1) +#define GITS_TYPER_Physical __BIT(0) + +#define GITS_CBASER_Valid __BIT(63) +#define GITS_CBASER_InnerCache __BITS(61,59) +#define GITS_CBASER_OuterCache __BITS(55,53) +#define GITS_CBASER_Physical_Address __BITS(51,12) +#define GITS_CBASER_Shareability __BITS(11,10) +#define GITS_CBASER_Size __BITS(7,0) + +#define GITS_CWRITER_Offset __BITS(19,5) +#define GITS_CWRITER_Retry __BIT(0) + +#define GITS_CREADR_Offset __BITS(19,5) +#define GITS_CREADR_Stalled __BIT(0) + +#define GITS_BASER_Valid __BIT(63) +#define GITS_BASER_Indirect __BIT(62) +#define GITS_BASER_InnerCache __BITS(61,59) +#define GITS_BASER_Type __BITS(58,56) +#define GITS_BASER_OuterCache __BITS(55,53) +#define GITS_BASER_Entry_Size __BITS(52,48) +#define GITS_BASER_Physical_Address __BITS(47,12) +#define GITS_BASER_Shareability __BITS(11,10) +#define GITS_BASER_Page_Size __BITS(9,8) +#define GITS_BASER_Size __BITS(7,0) + +#define GITS_Shareability_NS 0 // Non-shareable +#define GITS_Shareability_IS 1 // Inner Shareable +#define GITS_Shareability_OS 2 // Outer Shareable + +#define GITS_Cache_DEVICE_nGnRnE 0 // Device-nGnRnE +#define GITS_Cache_NORMAL_NC 1 // Non-cacheable +#define GITS_Cache_NORMAL_RA_WT 2 // Cacheable Read-allocate, Write-through +#define GITS_Cache_NORMAL_RA_WB 3 // Cacheable Read-allocate, Write-back +#define GITS_Cache_NORMAL_WA_WT 4 // Cacheable Write-allocate, Write-through +#define GITS_Cache_NORMAL_WA_WB 5 // Cacheable Write-allocate, Write-back +#define GITS_Cache_NORMAL_RA_WA_WT 6 // Cacheable Read-allocate, Write-allocate, Write-through +#define GITS_Cache_NORMAL_RA_WA_WB 7 // Cacheable Read-allocate, Write-allocate, Write-back + +#define GITS_Type_Unimplemented 0 // Unimplemented +#define GITS_Type_Devices 1 // Devices table +#define GITS_Type_vPEs 2 // vPEs table +#define GITS_Type_InterruptCollections 4 // Interrupt collections table + +#define GITS_Page_Size_4KB 0 +#define GITS_Page_Size_16KB 1 +#define GITS_Page_Size_64KB 2 + +struct gicv3_its_command { + uint64_t dw[4]; +}; + +#define GITS_CMD_MOVI 0x01 +#define GITS_CMD_INT 0x03 +#define GITS_CMD_CLEAR 0x04 +#define GITS_CMD_SYNC 0x05 +#define GITS_CMD_MAPD 0x08 +#define GITS_CMD_MAPC 0x09 +#define GITS_CMD_MAPTI 0x0A +#define GITS_CMD_MAPI 0x0B +#define GITS_CMD_INV 0x0C +#define GITS_CMD_INVALL 0x0D +#define GITS_CMD_MOVALL 0x0E +#define GITS_CMD_DISCARD 0x0F +#define GITS_CMD_VMOVI 0x21 +#define GITS_CMD_VMOVP 0x22 +#define GITS_CMD_VSYNC 0x25 +#define GITS_CMD_VMAPP 0x29 +#define GITS_CMD_VMAPTI 0x2A +#define GITS_CMD_VMAPI 0x2B +#define GITS_CMD_VINVALL 0x2D + #endif /* !_ARM_CORTEX_GICREG_H_ */ diff --git a/sys/arch/arm/cortex/gicv3.c b/sys/arch/arm/cortex/gicv3.c index b1566f122780..0e3a519f9e28 100644 --- a/sys/arch/arm/cortex/gicv3.c +++ b/sys/arch/arm/cortex/gicv3.c @@ -1,4 +1,4 @@ -/* $NetBSD: gicv3.c,v 1.4 2018/11/05 11:50:15 jmcneill Exp $ */ +/* $NetBSD: gicv3.c,v 1.5 2018/11/09 23:36:24 jmcneill Exp $ */ /*- * Copyright (c) 2018 Jared McNeill @@ -31,7 +31,7 @@ #define _INTR_PRIVATE #include -__KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.4 2018/11/05 11:50:15 jmcneill Exp $"); +__KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.5 2018/11/09 23:36:24 jmcneill Exp $"); #include #include @@ -49,6 +49,8 @@ __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.4 2018/11/05 11:50:15 jmcneill Exp $"); #define PICTOSOFTC(pic) \ ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) +#define LPITOSOFTC(lpi) \ + ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) #define IPL_TO_PRIORITY(ipl) ((IPL_HIGH - (ipl)) << 4) @@ -110,7 +112,7 @@ gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) if (group == 0) { sc->sc_enabled_sgippi |= mask; gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); - while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP) + while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) ; } else { gicd_write_4(sc, GICD_ISENABLERn(group), mask); @@ -129,7 +131,7 @@ gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) if (group == 0) { sc->sc_enabled_sgippi &= ~mask; gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); - while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP) + while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) ; } else { gicd_write_4(sc, GICD_ICENABLERn(group), mask); @@ -251,7 +253,7 @@ gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); /* Wait for register write to complete */ - while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP) + while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) ; /* Set default priorities */ @@ -286,7 +288,7 @@ gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); /* Wait for register write to complete */ - while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTRL) & GICR_CTRL_RWP) + while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) ; } @@ -457,11 +459,148 @@ static const struct pic_ops gicv3_picops = { #endif }; +static void +gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) +{ + struct gicv3_softc * const sc = LPITOSOFTC(pic); + int bit; + + while ((bit = ffs(mask)) != 0) { + sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; + mask &= ~__BIT(bit - 1); + } + + bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE); +} + +static void +gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) +{ + struct gicv3_softc * const sc = LPITOSOFTC(pic); + const u_int off = irqbase - pic->pic_irqbase; + int bit; + + while ((bit = ffs(mask)) != 0) { + sc->sc_lpiconf.base[off + bit - 1] &= ~GIC_LPICONF_Enable; + mask &= ~__BIT(bit - 1); + } + + bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, off, 32, BUS_DMASYNC_PREWRITE); +} + +static void +gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) +{ + struct gicv3_softc * const sc = LPITOSOFTC(pic); + + sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(is->is_ipl) | GIC_LPICONF_Res1; + + bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, is->is_irq, 1, BUS_DMASYNC_PREWRITE); +} + +static void +gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) +{ + struct gicv3_softc * const sc = LPITOSOFTC(pic); + struct gicv3_cpu_init *cpu_init; + uint32_t ctlr; + + /* If physical LPIs are not supported on this redistributor, just return. */ + const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); + if ((typer & GICR_TYPER_PLPIS) == 0) + return; + + /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ + sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); + + /* Disable LPIs before making changes */ + ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); + ctlr &= ~GICR_CTLR_Enable_LPIs; + gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); + arm_dsb(); + + /* Setup the LPI configuration table */ + const uint64_t propbase = sc->sc_lpiconf.segs[0].ds_addr | + __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | + __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability) | + __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); + gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); + + /* Setup the LPI pending table */ + const uint64_t pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | + __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability) | + __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache) | + GICR_PENDBASER_PTZ; + gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); + + /* Enable LPIs */ + ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); + ctlr |= GICR_CTLR_Enable_LPIs; + gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); + arm_dsb(); + + /* Setup ITS if present */ + LIST_FOREACH(cpu_init, &sc->sc_cpu_init, list) + cpu_init->func(cpu_init->arg, ci); +} + +static const struct pic_ops gicv3_lpiops = { + .pic_unblock_irqs = gicv3_lpi_unblock_irqs, + .pic_block_irqs = gicv3_lpi_block_irqs, + .pic_establish_irq = gicv3_lpi_establish_irq, +#ifdef MULTIPROCESSOR + .pic_cpu_init = gicv3_lpi_cpu_init, +#endif +}; + +void +gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) +{ + int nsegs, error; + + dma->len = len; + error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); + if (error) + panic("bus_dmamem_alloc failed: %d", error); + error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); + if (error) + panic("bus_dmamem_map failed: %d", error); + error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); + if (error) + panic("bus_dmamap_create failed: %d", error); + error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); + if (error) + panic("bus_dmamap_load failed: %d", error); + + memset(dma->base, 0, dma->len); + bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); +} + +static void +gicv3_lpi_init(struct gicv3_softc *sc) +{ + /* + * Allocate LPI configuration table + */ + gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); + KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); + + /* + * Allocate LPI pending tables + */ + const bus_size_t lpipend_sz = (sc->sc_lpi.pic_maxsources + sc->sc_lpi.pic_irqbase) / NBBY; + for (int cpuindex = 0; cpuindex < MAXCPUS; cpuindex++) { + gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); + KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); + } +} + void gicv3_irq_handler(void *frame) { struct cpu_info * const ci = curcpu(); struct gicv3_softc * const sc = gicv3_softc; + struct pic_softc *pic; const int oldipl = ci->ci_cpl; ci->ci_data.cpu_nintr++; @@ -472,10 +611,11 @@ gicv3_irq_handler(void *frame) if (irq == ICC_IAR_INTID_SPURIOUS) break; - if (irq >= sc->sc_pic.pic_maxsources) + pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; + if (irq - pic->pic_irqbase >= pic->pic_maxsources) continue; - struct intrsource * const is = sc->sc_pic.pic_sources[irq]; + struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; KASSERT(is != NULL); const int ipl = is->is_ipl; @@ -500,6 +640,8 @@ gicv3_init(struct gicv3_softc *sc) KASSERT(CPU_IS_PRIMARY(curcpu())); + LIST_INIT(&sc->sc_cpu_init); + sc->sc_pic.pic_ops = &gicv3_picops; sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer); snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); @@ -508,6 +650,15 @@ gicv3_init(struct gicv3_softc *sc) #endif pic_add(&sc->sc_pic, 0); + if ((gicd_typer & GICD_TYPER_LPIS) != 0) { + sc->sc_lpi.pic_ops = &gicv3_lpiops; + sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ + snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); + pic_add(&sc->sc_lpi, GIC_LPI_BASE); + + gicv3_lpi_init(sc); + } + KASSERT(gicv3_softc == NULL); gicv3_softc = sc; @@ -525,6 +676,8 @@ gicv3_init(struct gicv3_softc *sc) gicv3_dist_enable(sc); gicv3_cpu_init(&sc->sc_pic, curcpu()); + if ((gicd_typer & GICD_TYPER_LPIS) != 0) + gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); #ifdef __HAVE_PIC_FAST_SOFTINTS intr_establish(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO); diff --git a/sys/arch/arm/cortex/gicv3.h b/sys/arch/arm/cortex/gicv3.h index 4e15ed189517..8684785c571b 100644 --- a/sys/arch/arm/cortex/gicv3.h +++ b/sys/arch/arm/cortex/gicv3.h @@ -1,4 +1,4 @@ -/* $NetBSD: gicv3.h,v 1.1 2018/08/08 19:02:28 jmcneill Exp $ */ +/* $NetBSD: gicv3.h,v 1.2 2018/11/09 23:36:24 jmcneill Exp $ */ /*- * Copyright (c) 2018 Jared McNeill @@ -31,11 +31,27 @@ #include +struct gicv3_dma { + bus_dma_segment_t segs[1]; + bus_dmamap_t map; + uint8_t *base; + bus_size_t len; +}; + +struct gicv3_cpu_init { + void (*func)(void *, struct cpu_info *ci); + void *arg; + + LIST_ENTRY(gicv3_cpu_init) list; +}; + struct gicv3_softc { - struct pic_softc sc_pic; + struct pic_softc sc_pic; /* SGI/PPI/SGIs */ + struct pic_softc sc_lpi; /* LPIs */ device_t sc_dev; bus_space_tag_t sc_bst; + bus_dma_tag_t sc_dmat; bus_space_handle_t sc_bsh_d; /* GICD */ bus_space_handle_t *sc_bsh_r; /* GICR */ @@ -43,9 +59,22 @@ struct gicv3_softc { uint32_t sc_enabled_sgippi; uint64_t sc_default_irouter; + + /* LPI configuration table */ + struct gicv3_dma sc_lpiconf; + + /* LPI pending tables */ + struct gicv3_dma sc_lpipend[MAXCPUS]; + + /* Unique identifier for PEs */ + u_int sc_processor_id[MAXCPUS]; + + /* CPU init callbacks */ + LIST_HEAD(, gicv3_cpu_init) sc_cpu_init; }; int gicv3_init(struct gicv3_softc *); +void gicv3_dma_alloc(struct gicv3_softc *, struct gicv3_dma *, bus_size_t, bus_size_t); void gicv3_irq_handler(void *); #endif /* _ARM_CORTEX_GICV3_H */ diff --git a/sys/arch/arm/cortex/gicv3_its.c b/sys/arch/arm/cortex/gicv3_its.c new file mode 100644 index 000000000000..5e8e2ea676fd --- /dev/null +++ b/sys/arch/arm/cortex/gicv3_its.c @@ -0,0 +1,703 @@ +/* $NetBSD: gicv3_its.c,v 1.1 2018/11/09 23:36:24 jmcneill Exp $ */ + +/*- + * Copyright (c) 2018 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Jared McNeill . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define _INTR_PRIVATE + +#include +__KERNEL_RCSID(0, "$NetBSD: gicv3_its.c,v 1.1 2018/11/09 23:36:24 jmcneill Exp $"); + +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +/* + * ITS translation table sizes + */ +#define GITS_COMMANDS_SIZE 0x1000 +#define GITS_COMMANDS_ALIGN 0x10000 + +#define GITS_ITT_ALIGN 0x100 + +static inline uint32_t +gits_read_4(struct gicv3_its *its, bus_size_t reg) +{ + return bus_space_read_4(its->its_bst, its->its_bsh, reg); +} + +static inline void +gits_write_4(struct gicv3_its *its, bus_size_t reg, uint32_t val) +{ + bus_space_write_4(its->its_bst, its->its_bsh, reg, val); +} + +static inline uint64_t +gits_read_8(struct gicv3_its *its, bus_size_t reg) +{ + return bus_space_read_8(its->its_bst, its->its_bsh, reg); +} + +static inline void +gits_write_8(struct gicv3_its *its, bus_size_t reg, uint64_t val) +{ + bus_space_write_8(its->its_bst, its->its_bsh, reg, val); +} + +static inline void +gits_command(struct gicv3_its *its, const struct gicv3_its_command *cmd) +{ + uint64_t cwriter; + u_int woff; + + cwriter = gits_read_8(its, GITS_CWRITER); + woff = cwriter & GITS_CWRITER_Offset; + + memcpy(its->its_cmd.base + woff, cmd->dw, sizeof(cmd->dw)); + bus_dmamap_sync(its->its_dmat, its->its_cmd.map, woff, sizeof(cmd->dw), BUS_DMASYNC_PREWRITE); + + woff += sizeof(cmd->dw); + if (woff == its->its_cmd.len) + woff = 0; + + gits_write_8(its, GITS_CWRITER, woff); +} + +static inline void +gits_command_mapc(struct gicv3_its *its, uint16_t icid, uint64_t rdbase, bool v) +{ + struct gicv3_its_command cmd; + + KASSERT((rdbase & 0xffff) == 0); + + /* + * Map a collection table entry (ICID) to the target redistributor (RDbase). + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.dw[0] = GITS_CMD_MAPC; + cmd.dw[2] = icid; + if (v) { + cmd.dw[2] |= rdbase; + cmd.dw[2] |= __BIT(63); + } + + gits_command(its, &cmd); +} + +static inline void +gits_command_mapd(struct gicv3_its *its, uint32_t deviceid, uint64_t itt_addr, u_int size, bool v) +{ + struct gicv3_its_command cmd; + + KASSERT((itt_addr & 0xff) == 0); + + /* + * Map a device table entry (DeviceID) to its associated ITT (ITT_addr). + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.dw[0] = GITS_CMD_MAPD | ((uint64_t)deviceid << 32); + cmd.dw[1] = size; + if (v) { + cmd.dw[2] = itt_addr | __BIT(63); + } + + gits_command(its, &cmd); +} + +static inline void +gits_command_mapi(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid, uint16_t icid) +{ + struct gicv3_its_command cmd; + + /* + * Map the event defined by EventID and DeviceID into an ITT entry with ICID and pINTID = EventID + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.dw[0] = GITS_CMD_MAPI | ((uint64_t)deviceid << 32); + cmd.dw[1] = eventid; + cmd.dw[2] = icid; + + gits_command(its, &cmd); +} + +static inline void +gits_command_inv(struct gicv3_its *its, uint32_t deviceid, uint32_t eventid) +{ + struct gicv3_its_command cmd; + + /* + * Ensure any caching in the redistributors associated with the specified + * EventID is consistent with the LPI configuration tables. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.dw[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32); + cmd.dw[1] = eventid; + + gits_command(its, &cmd); +} + +static inline void +gits_command_invall(struct gicv3_its *its, uint16_t icid) +{ + struct gicv3_its_command cmd; + + /* + * Ensure any caching associated with this ICID is consistent with LPI + * configuration tables for all redistributors. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.dw[0] = GITS_CMD_INVALL; + cmd.dw[2] = icid; + + gits_command(its, &cmd); +} + +static inline void +gits_command_sync(struct gicv3_its *its, uint64_t rdbase) +{ + struct gicv3_its_command cmd; + + KASSERT((rdbase & 0xffff) == 0); + + /* + * Ensure all outstanding ITS operations associated with physical interrupts + * for the specified redistributor (RDbase) are globally observed before + * further ITS commands are executed. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.dw[0] = GITS_CMD_SYNC; + cmd.dw[2] = rdbase; + + gits_command(its, &cmd); +} + +static inline int +gits_wait(struct gicv3_its *its) +{ + u_int woff, roff; + int retry = 100000; + + /* + * The ITS command queue is empty when CWRITER and CREADR specify the + * same base address offset value. + */ + for (retry = 1000; retry > 0; retry--) { + woff = gits_read_8(its, GITS_CWRITER) & GITS_CWRITER_Offset; + roff = gits_read_8(its, GITS_CREADR) & GITS_CREADR_Offset; + if (woff == roff) + break; + delay(100); + } + if (retry == 0) { + device_printf(its->its_gic->sc_dev, "ITS command queue timeout\n"); + return ETIMEDOUT; + } + + return 0; +} + +static int +gicv3_its_msi_alloc_lpi(struct gicv3_its *its, + const struct pci_attach_args *pa) +{ + int n; + + for (n = 0; n < its->its_pic->pic_maxsources; n++) { + if (its->its_pa[n] == NULL) { + its->its_pa[n] = pa; + return n + its->its_pic->pic_irqbase; + } + } + + return -1; +} + +static void +gicv3_its_msi_free_lpi(struct gicv3_its *its, int lpi) +{ + KASSERT(lpi >= its->its_pic->pic_irqbase); + its->its_pa[lpi - its->its_pic->pic_irqbase] = NULL; +} + +static uint32_t +gicv3_its_devid(pci_chipset_tag_t pc, pcitag_t tag) +{ + int b, d, f; + + pci_decompose_tag(pc, tag, &b, &d, &f); + + return (b << 8) | (d << 3) | f; +} + +static struct gicv3_its_device * +gicv3_its_device_lookup(struct gicv3_its *its, uint32_t devid) +{ + struct gicv3_its_device *dev; + + LIST_FOREACH(dev, &its->its_devices, dev_list) + if (dev->dev_id == devid) + return dev; + + const uint64_t typer = gits_read_8(its, GITS_TYPER); + const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; + const u_int itt_entry_size = __SHIFTOUT(typer, GITS_TYPER_ITT_entry_size) + 1; + const u_int itt_size = roundup2(itt_entry_size * (1 << id_bits), GITS_ITT_ALIGN); + + dev = kmem_alloc(sizeof(*dev), KM_SLEEP); + dev->dev_id = devid; + gicv3_dma_alloc(its->its_gic, &dev->dev_itt, itt_size, GITS_ITT_ALIGN); + LIST_INSERT_HEAD(&its->its_devices, dev, dev_list); + + return dev; +} + +static void +gicv3_its_msi_enable(struct gicv3_its *its, int lpi) +{ + const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; + pci_chipset_tag_t pc = pa->pa_pc; + pcitag_t tag = pa->pa_tag; + pcireg_t ctl; + int off; + + if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) + panic("gicv3_its_msi_enable: device is not MSI-capable"); + + const uint64_t addr = its->its_base + GITS_TRANSLATER; + ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); + if (ctl & PCI_MSI_CTL_64BIT_ADDR) { + pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, + addr & 0xffffffff); + pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, + (addr >> 32) & 0xffffffff); + pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, lpi); + } else { + pci_conf_write(pc, tag, off + PCI_MSI_MADDR, + addr & 0xffffffff); + pci_conf_write(pc, tag, off + PCI_MSI_MDATA, lpi); + } + ctl |= PCI_MSI_CTL_MSI_ENABLE; + pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); +} + +static void +gicv3_its_msi_disable(struct gicv3_its *its, int lpi) +{ + const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; + pci_chipset_tag_t pc = pa->pa_pc; + pcitag_t tag = pa->pa_tag; + pcireg_t ctl; + int off; + + if (!pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL)) + panic("gicv3_its_msi_enable: device is not MSI-capable"); + + ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); + ctl &= ~PCI_MSI_CTL_MSI_ENABLE; + pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); +} + +static void +gicv3_its_msix_enable(struct gicv3_its *its, int lpi, int msix_vec, + bus_space_tag_t bst, bus_space_handle_t bsh) +{ + const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; + pci_chipset_tag_t pc = pa->pa_pc; + pcitag_t tag = pa->pa_tag; + pcireg_t ctl; + int off; + + if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) + panic("gicv3_its_msix_enable: device is not MSI-X-capable"); + + const uint64_t addr = its->its_base + GITS_TRANSLATER; + const uint64_t entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; + bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, (uint32_t)addr); + bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, (uint32_t)(addr >> 32)); + bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_DATA, lpi); + bus_space_write_4(bst, bsh, entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0); + + ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); + ctl |= PCI_MSIX_CTL_ENABLE; + pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); +} + +static void +gicv3_its_msix_disable(struct gicv3_its *its, int lpi) +{ + const struct pci_attach_args *pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; + pci_chipset_tag_t pc = pa->pa_pc; + pcitag_t tag = pa->pa_tag; + pcireg_t ctl; + int off; + + if (!pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL)) + panic("gicv3_its_msix_disable: device is not MSI-X-capable"); + + ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); + ctl &= ~PCI_MSIX_CTL_ENABLE; + pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); +} + +static pci_intr_handle_t * +gicv3_its_msi_alloc(struct arm_pci_msi *msi, int *count, + const struct pci_attach_args *pa, bool exact) +{ + struct gicv3_its * const its = msi->msi_priv; + struct gicv3_its_device *dev; + struct cpu_info *ci = curcpu(); /* XXX */ + pci_intr_handle_t *vectors; + int n, off; + + if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSI, &off, NULL)) + return NULL; + + const uint64_t typer = gits_read_8(its, GITS_TYPER); + const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; + if (*count == 0 || *count > (1 << id_bits)) + return NULL; + + const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); + + /* + * Map device + */ + dev = gicv3_its_device_lookup(its, devid); + gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits - 1, true); + gits_wait(its); + + vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); + for (n = 0; n < *count; n++) { + const int lpi = gicv3_its_msi_alloc_lpi(its, pa); + vectors[n] = ARM_PCI_INTR_MSI | + __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) | + __SHIFTIN(n, ARM_PCI_INTR_MSI_VEC) | + __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); + + gicv3_its_msi_enable(its, lpi); + + /* + * Map event + */ + gits_command_mapi(its, devid, lpi, cpu_index(ci)); + gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); + } + gits_wait(its); + + return vectors; +} + +static pci_intr_handle_t * +gicv3_its_msix_alloc(struct arm_pci_msi *msi, u_int *table_indexes, int *count, + const struct pci_attach_args *pa, bool exact) +{ + struct gicv3_its * const its = msi->msi_priv; + struct gicv3_its_device *dev; + struct cpu_info *ci = curcpu(); /* XXX */ + pci_intr_handle_t *vectors; + bus_space_tag_t bst; + bus_space_handle_t bsh; + bus_size_t bsz; + uint32_t table_offset, table_size; + int n, off, bar, error; + pcireg_t tbl; + + if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &off, NULL)) + return NULL; + + const uint64_t typer = gits_read_8(its, GITS_TYPER); + const u_int id_bits = __SHIFTOUT(typer, GITS_TYPER_ID_bits) + 1; + if (*count == 0 || *count > (1 << id_bits)) + return NULL; + + tbl = pci_conf_read(pa->pa_pc, pa->pa_tag, off + PCI_MSIX_TBLOFFSET); + bar = PCI_BAR0 + (4 * (tbl & PCI_MSIX_PBABIR_MASK)); + table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; + table_size = pci_msix_count(pa->pa_pc, pa->pa_tag) * PCI_MSIX_TABLE_ENTRY_SIZE; + if (table_size == 0) + return NULL; + + error = pci_mapreg_submap(pa, bar, pci_mapreg_type(pa->pa_pc, pa->pa_tag, bar), + BUS_SPACE_MAP_LINEAR, roundup(table_size, PAGE_SIZE), table_offset, + &bst, &bsh, NULL, &bsz); + if (error) + return NULL; + + const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); + + /* + * Map device + */ + dev = gicv3_its_device_lookup(its, devid); + gits_command_mapd(its, devid, dev->dev_itt.segs[0].ds_addr, id_bits - 1, true); + gits_wait(its); + + vectors = kmem_alloc(sizeof(*vectors) * *count, KM_SLEEP); + for (n = 0; n < *count; n++) { + const int lpi = gicv3_its_msi_alloc_lpi(its, pa); + const int msix_vec = table_indexes ? table_indexes[n] : n; + vectors[msix_vec] = ARM_PCI_INTR_MSIX | + __SHIFTIN(lpi, ARM_PCI_INTR_IRQ) | + __SHIFTIN(msix_vec, ARM_PCI_INTR_MSI_VEC) | + __SHIFTIN(msi->msi_id, ARM_PCI_INTR_FRAME); + + gicv3_its_msix_enable(its, lpi, msix_vec, bst, bsh); + + /* + * Map event + */ + gits_command_mapi(its, devid, lpi, cpu_index(ci)); + gits_command_sync(its, its->its_rdbase[cpu_index(ci)]); + } + gits_wait(its); + + bus_space_unmap(bst, bsh, bsz); + + return vectors; +} + +static void * +gicv3_its_msi_intr_establish(struct arm_pci_msi *msi, + pci_intr_handle_t ih, int ipl, int (*func)(void *), void *arg) +{ + struct gicv3_its * const its = msi->msi_priv; + const struct pci_attach_args *pa; + void *intrh; + + const int lpi = __SHIFTOUT(ih, ARM_PCI_INTR_IRQ); + const int mpsafe = (ih & ARM_PCI_INTR_MPSAFE) ? IST_MPSAFE : 0; + + intrh = pic_establish_intr(its->its_pic, lpi - its->its_pic->pic_irqbase, ipl, + IST_EDGE | mpsafe, func, arg); + if (intrh == NULL) + return NULL; + + /* Invalidate LPI configuration tables */ + pa = its->its_pa[lpi - its->its_pic->pic_irqbase]; + KASSERT(pa != NULL); + const uint32_t devid = gicv3_its_devid(pa->pa_pc, pa->pa_tag); + gits_command_inv(its, devid, lpi); + + return intrh; +} + +static void +gicv3_its_msi_intr_release(struct arm_pci_msi *msi, pci_intr_handle_t *pih, + int count) +{ + struct gicv3_its * const its = msi->msi_priv; + int n; + + for (n = 0; n < count; n++) { + const int lpi = __SHIFTOUT(pih[n], ARM_PCI_INTR_IRQ); + KASSERT(lpi >= its->its_pic->pic_irqbase); + if (pih[n] & ARM_PCI_INTR_MSIX) + gicv3_its_msix_disable(its, lpi); + if (pih[n] & ARM_PCI_INTR_MSI) + gicv3_its_msi_disable(its, lpi); + gicv3_its_msi_free_lpi(its, lpi); + struct intrsource * const is = + its->its_pic->pic_sources[lpi - its->its_pic->pic_irqbase]; + if (is != NULL) + pic_disestablish_source(is); + } +} + +static void +gicv3_its_command_init(struct gicv3_softc *sc, struct gicv3_its *its) +{ + uint64_t cbaser; + + gicv3_dma_alloc(sc, &its->its_cmd, GITS_COMMANDS_SIZE, GITS_COMMANDS_ALIGN); + + cbaser = its->its_cmd.segs[0].ds_addr; + cbaser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_CBASER_InnerCache); + cbaser |= __SHIFTIN(GITS_Shareability_NS, GITS_CBASER_Shareability); + cbaser |= __SHIFTIN((its->its_cmd.len / 4096) - 1, GITS_CBASER_Size); + cbaser |= GITS_CBASER_Valid; + + gits_write_8(its, GITS_CBASER, cbaser); + gits_write_8(its, GITS_CWRITER, 0); +} + +static void +gicv3_its_table_init(struct gicv3_softc *sc, struct gicv3_its *its) +{ + u_int table_size, page_size, table_align; + uint64_t baser; + int tab; + + const uint64_t typer = gits_read_8(its, GITS_TYPER); + const u_int devbits = __SHIFTOUT(typer, GITS_TYPER_Devbits) + 1; + + for (tab = 0; tab < 8; tab++) { + baser = gits_read_8(its, GITS_BASERn(tab)); + + const u_int entry_size = __SHIFTOUT(baser, GITS_BASER_Entry_Size) + 1; + + switch (__SHIFTOUT(baser, GITS_BASER_Page_Size)) { + case GITS_Page_Size_4KB: + page_size = 4096; + table_align = 4096; + break; + case GITS_Page_Size_16KB: + page_size = 16384; + table_align = 4096; + break; + case GITS_Page_Size_64KB: + default: + page_size = 65536; + table_align = 65536; + break; + } + + switch (__SHIFTOUT(baser, GITS_BASER_Type)) { + case GITS_Type_Devices: + /* + * Table size scales with the width of the DeviceID. + */ + table_size = roundup(entry_size * (1 << devbits), page_size); + break; + case GITS_Type_InterruptCollections: + /* + * Allocate space for one interrupt collection per CPU. + */ + table_size = roundup(entry_size * MAXCPUS, page_size); + break; + default: + table_size = 0; + break; + } + + if (table_size == 0) + continue; + + aprint_normal_dev(sc->sc_dev, "ITS TT%u type %#x size %#x\n", tab, (u_int)__SHIFTOUT(baser, GITS_BASER_Type), table_size); + gicv3_dma_alloc(sc, &its->its_tab[tab], table_size, table_align); + + baser &= ~GITS_BASER_Size; + baser |= __SHIFTIN(table_size / page_size - 1, GITS_BASER_Size); + baser &= ~GITS_BASER_Physical_Address; + baser |= its->its_tab[tab].segs[0].ds_addr; + baser &= ~GITS_BASER_InnerCache; + baser |= __SHIFTIN(GITS_Cache_NORMAL_NC, GITS_BASER_InnerCache); + baser &= ~GITS_BASER_Shareability; + baser |= __SHIFTIN(GITS_Shareability_NS, GITS_BASER_Shareability); + baser |= GITS_BASER_Valid; + + gits_write_8(its, GITS_BASERn(tab), baser); + } +} + +static void +gicv3_its_enable(struct gicv3_softc *sc, struct gicv3_its *its) +{ + uint32_t ctlr; + + ctlr = gits_read_4(its, GITS_CTLR); + ctlr |= GITS_CTLR_Enabled; + gits_write_4(its, GITS_CTLR, ctlr); +} + +static void +gicv3_its_cpu_init(void *priv, struct cpu_info *ci) +{ + struct gicv3_its * const its = priv; + struct gicv3_softc * const sc = its->its_gic; + uint64_t rdbase; + + const uint64_t typer = bus_space_read_8(sc->sc_bst, its->its_bsh, GITS_TYPER); + if (typer & GITS_TYPER_PTA) { + void *va = bus_space_vaddr(sc->sc_bst, sc->sc_bsh_r[ci->ci_gic_redist]); + rdbase = vtophys((vaddr_t)va); + } else { + rdbase = (uint64_t)sc->sc_processor_id[cpu_index(ci)] << 16; + } + its->its_rdbase[cpu_index(ci)] = rdbase; + + /* + * Map collection ID of this CPU's index to this CPU's redistributor. + */ + gits_command_mapc(its, cpu_index(ci), rdbase, true); + gits_command_invall(its, cpu_index(ci)); + gits_wait(its); +} + +int +gicv3_its_init(struct gicv3_softc *sc, bus_space_handle_t bsh, + uint64_t its_base, uint32_t its_id) +{ + struct gicv3_its *its; + struct arm_pci_msi *msi; + + const uint64_t typer = bus_space_read_8(sc->sc_bst, bsh, GITS_TYPER); + if ((typer & GITS_TYPER_Physical) == 0) + return ENXIO; + + its = kmem_alloc(sizeof(*its), KM_SLEEP); + its->its_id = its_id; + its->its_bst = sc->sc_bst; + its->its_bsh = bsh; + its->its_dmat = sc->sc_dmat; + its->its_base = its_base; + its->its_pic = &sc->sc_lpi; + KASSERT(its->its_pic->pic_maxsources > 0); + its->its_pa = kmem_zalloc(sizeof(struct pci_attach_args *) * its->its_pic->pic_maxsources, KM_SLEEP); + its->its_gic = sc; + its->its_init.func = gicv3_its_cpu_init; + its->its_init.arg = its; + LIST_INIT(&its->its_devices); + LIST_INSERT_HEAD(&sc->sc_cpu_init, &its->its_init, list); + + gicv3_its_command_init(sc, its); + gicv3_its_table_init(sc, its); + + gicv3_its_enable(sc, its); + + gicv3_its_cpu_init(its, curcpu()); + + msi = &its->its_msi; + msi->msi_dev = sc->sc_dev; + msi->msi_priv = its; + msi->msi_alloc = gicv3_its_msi_alloc; + msi->msix_alloc = gicv3_its_msix_alloc; + msi->msi_intr_establish = gicv3_its_msi_intr_establish; + msi->msi_intr_release = gicv3_its_msi_intr_release; + + return arm_pci_msi_add(msi); +} diff --git a/sys/arch/arm/cortex/gicv3_its.h b/sys/arch/arm/cortex/gicv3_its.h new file mode 100644 index 000000000000..9d3ae67ff5df --- /dev/null +++ b/sys/arch/arm/cortex/gicv3_its.h @@ -0,0 +1,70 @@ +/* $NetBSD: gicv3_its.h,v 1.1 2018/11/09 23:36:24 jmcneill Exp $ */ + +/*- + * Copyright (c) 2018 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Jared McNeill . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARM_CORTEX_GICV3_ITS_H +#define _ARM_CORTEX_GICV3_ITS_H + +#include +#include +#include + +struct gicv3_its_device { + uint32_t dev_id; + struct gicv3_dma dev_itt; + + LIST_ENTRY(gicv3_its_device) dev_list; +}; + +struct gicv3_its { + bus_space_tag_t its_bst; + bus_space_handle_t its_bsh; + bus_dma_tag_t its_dmat; + uint32_t its_id; + uint64_t its_base; + uint64_t its_rdbase[MAXCPUS]; + + struct gicv3_softc *its_gic; + struct gicv3_cpu_init its_init; + + struct pic_softc *its_pic; + const struct pci_attach_args **its_pa; + + LIST_HEAD(, gicv3_its_device) its_devices; + + struct gicv3_dma its_cmd; /* Command queue */ + struct gicv3_dma its_tab[8]; /* ITS tables */ + + struct arm_pci_msi its_msi; +}; + +int gicv3_its_init(struct gicv3_softc *, bus_space_handle_t, uint64_t, uint32_t); + +#endif /* !_ARM_CORTEX_GICV3_ITS_H */