diff --git a/sys/arch/arm/pic/files.pic b/sys/arch/arm/pic/files.pic index feca067c83ca..64155131ab14 100644 --- a/sys/arch/arm/pic/files.pic +++ b/sys/arch/arm/pic/files.pic @@ -1,9 +1,11 @@ -# $NetBSD: files.pic,v 1.3 2011/03/11 03:16:14 bsh Exp $ +# $NetBSD: files.pic,v 1.4 2012/09/01 00:00:42 matt Exp $ # # Configuration info for the common PIC code. # define pic define pic_splfuncs +defflag opt_arm_intr_impl.h __HAVE_PIC_SET_PRIORITY +defflag opt_arm_intr_impl.h __HAVE_PIC_SOFTINT file arch/arm/pic/pic.c pic file arch/arm/pic/pic_splfuncs.c pic & pic_splfuncs diff --git a/sys/arch/arm/pic/pic.c b/sys/arch/arm/pic/pic.c index 37d86ec1e550..fddf4ace8f44 100644 --- a/sys/arch/arm/pic/pic.c +++ b/sys/arch/arm/pic/pic.c @@ -1,4 +1,4 @@ -/* $NetBSD: pic.c,v 1.12 2012/07/20 21:53:57 matt Exp $ */ +/* $NetBSD: pic.c,v 1.13 2012/09/01 00:00:42 matt Exp $ */ /*- * Copyright (c) 2008 The NetBSD Foundation, Inc. * All rights reserved. @@ -28,15 +28,17 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.12 2012/07/20 21:53:57 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.13 2012/09/01 00:00:42 matt Exp $"); #define _INTR_PRIVATE #include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include #include @@ -74,10 +76,80 @@ EVCNT_ATTACH_STATIC(pic_deferral_ev); void pic_set_priority(struct cpu_info *ci, int newipl) { - register_t psw = disable_interrupts(I32_bit); + register_t psw = cpsid(I32_bit); + if (pic_list[0] != NULL) + (pic_list[0]->pic_ops->pic_set_priority)(pic_list[0], newipl); ci->ci_cpl = newipl; - (pic_list[0]->pic_set_priority)(newipl); - restore_interrupts(psw); + if ((psw & I32_bit) == 0) + cpsie(I32_bit); +} +#endif + +#ifdef MULTIPROCESSOR +int +pic_ipi_nop(void *arg) +{ + /* do nothing */ + return 1; +} + +int +pic_ipi_xcall(void *arg) +{ + xc_ipi_handler(); + return 1; +} + +void +intr_cpu_init(struct cpu_info *ci) +{ + for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { + struct pic_softc * const pic = pic_list[slot]; + if (pic != NULL && pic->pic_ops->pic_cpu_init != NULL) { + (*pic->pic_ops->pic_cpu_init)(pic, ci); + } + } +} + +typedef void (*pic_ipi_send_func_t)(struct pic_softc *, u_long); + +static struct pic_softc * +pic_ipi_sender(void) +{ + for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { + struct pic_softc * const pic = pic_list[slot]; + if (pic != NULL && pic->pic_ops->pic_ipi_send != NULL) { + return pic; + } + } + return NULL; +} + +void +intr_ipi_send(const kcpuset_t *kcp, u_long ipi) +{ + struct pic_softc * const pic = pic_ipi_sender(); + KASSERT(ipi < NIPI); + if (cold && pic == NULL) + return; + KASSERT(pic != NULL); + (*pic->pic_ops->pic_ipi_send)(pic, kcp, ipi); +} +#endif /* MULTIPROCESSOR */ + +#ifdef __HAVE_PIC_FAST_SOFTINTS +int +pic_handle_softint(void *arg) +{ + void softint_switch(lwp_t *, int); + struct cpu_info * const ci = curcpu(); + const size_t softint = (size_t) arg; + int s = splhigh(); + ci->ci_intr_depth--; // don't count these as interrupts + softint_switch(ci->ci_softlwps[softint], s); + ci->ci_intr_depth++; + splx(s); + return 1; } #endif @@ -185,6 +257,7 @@ pic_dispatch(struct intrsource *is, void *frame) { int rv; + if (__predict_false(is->is_arg == NULL) && __predict_true(frame != NULL)) { rv = (*is->is_func)(frame); @@ -194,7 +267,11 @@ pic_dispatch(struct intrsource *is, void *frame) pic_deferral_ev.ev_count++; return; } - is->is_ev.ev_count++; + + struct pic_percpu * const pcpu = percpu_getref(is->is_pic->pic_percpu); + KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); + pcpu->pcpu_evs[is->is_irq].ev_count++; + percpu_putref(is->is_pic->pic_percpu); } void @@ -367,8 +444,10 @@ void pic_do_pending_ints(register_t psw, int newipl, void *frame) { struct cpu_info * const ci = curcpu(); - if (__predict_false(newipl == IPL_HIGH)) + if (__predict_false(newipl == IPL_HIGH)) { + KASSERTMSG(ci->ci_cpl == IPL_HIGH, "cpl %d", ci->ci_cpl); return; + } while ((pic_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) { KASSERT(pic_pending_ipls < __BIT(NIPL)); for (;;) { @@ -384,8 +463,34 @@ pic_do_pending_ints(register_t psw, int newipl, void *frame) } if (ci->ci_cpl != newipl) pic_set_priority(ci, newipl); -#ifdef __HAVE_FAST_SOFTINTS - cpu_dosoftints(); +} + +static void +pic_percpu_allocate(void *v0, void *v1, struct cpu_info *ci) +{ + struct pic_percpu * const pcpu = v0; + struct pic_softc * const pic = v1; + + pcpu->pcpu_evs = kmem_zalloc(pic->pic_maxsources * sizeof(pcpu->pcpu_evs[0]), + KM_SLEEP); + KASSERT(pcpu->pcpu_evs != NULL); + +#define PCPU_NAMELEN 32 + const size_t namelen = strlen(pic->pic_name) + 4 + strlen(ci->ci_data.cpu_name); + + KASSERT(namelen < PCPU_NAMELEN); + pcpu->pcpu_name = kmem_alloc(PCPU_NAMELEN, KM_SLEEP); +#ifdef MULTIPROCESSOR + snprintf(pcpu->pcpu_name, PCPU_NAMELEN, + "%s (%s)", pic->pic_name, ci->ci_data.cpu_name); +#else + strlcpy(pcpu->pcpu_name, pic->pic_name, PCPU_NAMELEN); +#endif + pcpu->pcpu_magic = PICPERCPU_MAGIC; +#if 0 + printf("%s: %s %s: <%s>\n", + __func__, ci->ci_data.cpu_name, pic->pic_name, + pcpu->pcpu_name); #endif } @@ -394,6 +499,8 @@ pic_add(struct pic_softc *pic, int irqbase) { int slot, maybe_slot = -1; + KASSERT(strlen(pic->pic_name) > 0); + for (slot = 0; slot < PIC_MAXPICS; slot++) { struct pic_softc * const xpic = pic_list[slot]; if (xpic == NULL) { @@ -422,10 +529,32 @@ pic_add(struct pic_softc *pic, int irqbase) KASSERT(pic->pic_maxsources <= PIC_MAXSOURCES); KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES); + /* + * Allocate a pointer to each cpu's evcnts and then, for each cpu, + * allocate its evcnts and then attach an evcnt for each pin. + * We can't allocate the evcnt structures directly since + * percpu will move the contents of percpu memory around and + * corrupt the pointers in the evcnts themselves. Remember, any + * problem can be solved with sufficient indirection. + */ + pic->pic_percpu = percpu_alloc(sizeof(struct pic_percpu)); + KASSERT(pic->pic_percpu != NULL); + + /* + * Now allocate the per-cpu evcnts. + */ + percpu_foreach(pic->pic_percpu, pic_percpu_allocate, pic); + pic->pic_sources = &pic_sources[pic_sourcebase]; pic->pic_irqbase = irqbase; pic_sourcebase += pic->pic_maxsources; pic->pic_id = slot; +#ifdef __HAVE_PIC_SET_PRIORITY + KASSERT((slot == 0) == (pic->pic_ops->pic_set_priority != NULL)); +#endif +#ifdef MULTIPROCESSOR + KASSERT((slot == 0) == (pic->pic_ops->pic_ipi_send != NULL)); +#endif pic_list[slot] = pic; } @@ -442,6 +571,17 @@ pic_alloc_irq(struct pic_softc *pic) return -1; } +static void +pic_percpu_evcnt_attach(void *v0, void *v1, struct cpu_info *ci) +{ + struct pic_percpu * const pcpu = v0; + struct intrsource * const is = v1; + + KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); + evcnt_attach_dynamic(&pcpu->pcpu_evs[is->is_irq], EVCNT_TYPE_INTR, NULL, + pcpu->pcpu_name, is->is_source); +} + void * pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, int (*func)(void *), void *arg) @@ -465,15 +605,17 @@ pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, is->is_type = type; is->is_func = func; is->is_arg = arg; - + if (pic->pic_ops->pic_source_name) (*pic->pic_ops->pic_source_name)(pic, irq, is->is_source, sizeof(is->is_source)); else snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq); - evcnt_attach_dynamic(&is->is_ev, EVCNT_TYPE_INTR, NULL, - pic->pic_name, is->is_source); + /* + * Now attach the per-cpu evcnts. + */ + percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_attach, is); pic->pic_sources[irq] = is; @@ -522,16 +664,31 @@ pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, return is; } +static void +pic_percpu_evcnt_deattach(void *v0, void *v1, struct cpu_info *ci) +{ + struct pic_percpu * const pcpu = v0; + struct intrsource * const is = v1; + + KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); + evcnt_detach(&pcpu->pcpu_evs[is->is_irq]); +} + void pic_disestablish_source(struct intrsource *is) { struct pic_softc * const pic = is->is_pic; const int irq = is->is_irq; + KASSERT(is == pic->pic_sources[irq]); + (*pic->pic_ops->pic_block_irqs)(pic, irq & ~31, __BIT(irq)); pic->pic_sources[irq] = NULL; pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL; - evcnt_detach(&is->is_ev); + /* + * Now detach the per-cpu evcnts. + */ + percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_deattach, is); kmem_free(is, sizeof(*is)); } @@ -539,12 +696,10 @@ pic_disestablish_source(struct intrsource *is) void * intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg) { - int slot; - KASSERT(!cpu_intr_p()); KASSERT(!cpu_softintr_p()); - for (slot = 0; slot < PIC_MAXPICS; slot++) { + for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { struct pic_softc * const pic = pic_list[slot]; if (pic == NULL || pic->pic_irqbase < 0) continue; @@ -562,5 +717,9 @@ void intr_disestablish(void *ih) { struct intrsource * const is = ih; + + KASSERT(!cpu_intr_p()); + KASSERT(!cpu_softintr_p()); + pic_disestablish_source(is); } diff --git a/sys/arch/arm/pic/pic_splfuncs.c b/sys/arch/arm/pic/pic_splfuncs.c index 66bcd4303764..1e80bf83a47e 100644 --- a/sys/arch/arm/pic/pic_splfuncs.c +++ b/sys/arch/arm/pic/pic_splfuncs.c @@ -1,4 +1,4 @@ -/* $NetBSD: pic_splfuncs.c,v 1.3 2012/07/14 07:52:53 matt Exp $ */ +/* $NetBSD: pic_splfuncs.c,v 1.4 2012/09/01 00:00:42 matt Exp $ */ /*- * Copyright (c) 2008 The NetBSD Foundation, Inc. * All rights reserved. @@ -28,15 +28,15 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.3 2012/07/14 07:52:53 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pic_splfuncs.c,v 1.4 2012/09/01 00:00:42 matt Exp $"); #define _INTR_PRIVATE #include +#include #include -#include -#include -#include -#include +#include + +#include #include #include @@ -63,11 +63,13 @@ _spllower(int newipl) const int oldipl = ci->ci_cpl; KASSERT(panicstr || newipl <= ci->ci_cpl); if (newipl < ci->ci_cpl) { - register_t psw = disable_interrupts(I32_bit); + register_t psw = cpsid(I32_bit); ci->ci_intr_depth++; pic_do_pending_ints(psw, newipl, NULL); ci->ci_intr_depth--; - restore_interrupts(psw); + if ((psw & I32_bit) == 0) + cpsie(I32_bit); + cpu_dosoftints(); } return oldipl; } @@ -77,12 +79,26 @@ splx(int savedipl) { struct cpu_info * const ci = curcpu(); KASSERT(savedipl < NIPL); - if (savedipl < ci->ci_cpl) { - register_t psw = disable_interrupts(I32_bit); - ci->ci_intr_depth++; - pic_do_pending_ints(psw, savedipl, NULL); - ci->ci_intr_depth--; - restore_interrupts(psw); + + if (__predict_false(savedipl == ci->ci_cpl)) { + return; } - ci->ci_cpl = savedipl; + + register_t psw = cpsid(I32_bit); + KASSERTMSG(panicstr != NULL || savedipl < ci->ci_cpl, + "splx(%d) to a higher ipl than %d", savedipl, ci->ci_cpl); + + ci->ci_intr_depth++; + pic_do_pending_ints(psw, savedipl, NULL); + ci->ci_intr_depth--; + KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d", + ci->ci_cpl, savedipl); + cpu_dosoftints(); + KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d", + ci->ci_cpl, savedipl); + if ((psw & I32_bit) == 0) + cpsie(I32_bit); + + KASSERTMSG(ci->ci_cpl == savedipl, "cpl %d savedipl %d", + ci->ci_cpl, savedipl); } diff --git a/sys/arch/arm/pic/picvar.h b/sys/arch/arm/pic/picvar.h index b7c628e23fe4..f4eb83d85fea 100644 --- a/sys/arch/arm/pic/picvar.h +++ b/sys/arch/arm/pic/picvar.h @@ -1,4 +1,4 @@ -/* $NetBSD: picvar.h,v 1.6 2012/07/14 07:52:53 matt Exp $ */ +/* $NetBSD: picvar.h,v 1.7 2012/09/01 00:00:42 matt Exp $ */ /*- * Copyright (c) 2008 The NetBSD Foundation, Inc. * All rights reserved. @@ -30,6 +30,10 @@ #ifndef _ARM_PIC_PICVAR_H_ #define _ARM_PIC_PICVAR_H_ +#ifdef MULTIPROCESSOR +#include +#endif + int _splraise(int); int _spllower(int); void splx(int); @@ -38,6 +42,17 @@ const char * struct pic_softc; struct intrsource; +struct cpu_info; + +#define IPI_AST 0 /* just get an interrupt */ +#define IPI_XCALL 1 /* xcall */ +#define IPI_NOP 2 /* just get an interrupt (armv6) */ +#ifndef __HAVE_PREEMPTION +#define NIPI 3 +#else +#define IPI_KPREEMPT 4 /* cause a preemption */ +#define NIPI 4 +#endif int pic_handle_intr(void *); void pic_mark_pending(struct pic_softc *pic, int irq); @@ -54,10 +69,17 @@ void pic_dispatch(struct intrsource *is, void *frame); void *intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg); void intr_disestablish(void *); +#ifdef MULTIPROCESSOR +void intr_cpu_init(struct cpu_info *); +void intr_ipi_send(const kcpuset_t *, u_long ipi); +#endif #ifdef _INTR_PRIVATE +#include "opt_arm_intr_impl.h" + #include +#include #ifndef PIC_MAXPICS #define PIC_MAXPICS 32 @@ -77,10 +99,17 @@ struct intrsource { uint8_t is_ipl; /* IPL_xxx */ uint8_t is_irq; /* local to pic */ uint8_t is_iplidx; - struct evcnt is_ev; char is_source[16]; }; +struct pic_percpu { + struct evcnt *pcpu_evs; + char *pcpu_name; + uint32_t pcpu_magic; +}; + +#define PICPERCPU_MAGIC 0xfeedface + struct pic_softc { const struct pic_ops *pic_ops; struct intrsource **pic_sources; @@ -88,6 +117,7 @@ struct pic_softc { volatile uint32_t pic_blocked_irqs[(PIC_MAXSOURCES + 31) / 32]; volatile uint32_t pic_pending_ipls; size_t pic_maxsources; + percpu_t *pic_percpu; uint8_t pic_id; int16_t pic_irqbase; char pic_name[14]; @@ -104,6 +134,10 @@ struct pic_ops { #ifdef __HAVE_PIC_SET_PRIORITY void (*pic_set_priority)(struct pic_softc *, int); #endif +#ifdef MULTIPROCESSOR + void (*pic_cpu_init)(struct pic_softc *, struct cpu_info *); + void (*pic_ipi_send)(struct pic_softc *, const kcpuset_t *, u_long); +#endif }; #ifdef __HAVE_PIC_SET_PRIORITY @@ -120,6 +154,13 @@ void pic_set_priority(struct cpu_info *, int); void pic_add(struct pic_softc *, int); void pic_do_pending_int(void); +#ifdef MULTIPROCESSOR +int pic_ipi_nop(void *); +int pic_ipi_xcall(void *); +#endif +#ifdef __HAVE_PIC_FAST_SOFTINTS +int pic_handle_softint(void *); +#endif extern struct pic_softc * pic_list[PIC_MAXPICS]; #endif /* _INTR_PRIVATE */