Improve IPI handling:
- make IPI takes two arguments. - add IPI event counters per-CPU. - implement IPI functions which were missing or broken. - insert DELAY while halting primary CPU in IPI handler.
This commit is contained in:
nakayama
parent
607ead0ef4
commit
ead4e6f7b9
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: cpu.h,v 1.75 2008/03/02 15:28:26 nakayama Exp $ */
|
||||
/* $NetBSD: cpu.h,v 1.76 2008/03/14 15:38:00 nakayama Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1992, 1993
|
||||
|
|
@ -128,7 +128,12 @@ struct cpu_info {
|
|||
/* Interrupts */
|
||||
struct intrhand *ci_intrpending[16];
|
||||
struct intrhand *ci_intrlev0;
|
||||
|
||||
/* Event counters */
|
||||
struct evcnt ci_tick_evcnt;
|
||||
#ifdef MULTIPROCESSOR
|
||||
struct evcnt ci_ipi_evcnt[IPI_EVCNT_NUM];
|
||||
#endif
|
||||
|
||||
int ci_flags;
|
||||
int ci_want_ast;
|
||||
|
|
@ -238,9 +243,9 @@ void cpu_boot_secondary_processors(void);
|
|||
*/
|
||||
typedef void (* ipifunc_t)(void *);
|
||||
|
||||
void sparc64_multicast_ipi (sparc64_cpuset_t, ipifunc_t, uint64_t);
|
||||
void sparc64_broadcast_ipi (ipifunc_t, uint64_t);
|
||||
void sparc64_send_ipi (int, ipifunc_t, uint64_t);
|
||||
void sparc64_multicast_ipi(sparc64_cpuset_t, ipifunc_t, uint64_t, uint64_t);
|
||||
void sparc64_broadcast_ipi(ipifunc_t, uint64_t, uint64_t);
|
||||
void sparc64_send_ipi(int, ipifunc_t, uint64_t, uint64_t);
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
|
@ -320,10 +325,10 @@ int isbad(struct dkbad *bt, int, int, int);
|
|||
void * reserve_dumppages(void *);
|
||||
/* clock.c */
|
||||
struct timeval;
|
||||
int tickintr(void *); /* level 10 (tick) interrupt code */
|
||||
int tickintr(void *); /* level 10/14 (tick) interrupt code */
|
||||
int clockintr(void *); /* level 10 (clock) interrupt code */
|
||||
int statintr(void *); /* level 14 (statclock) interrupt code */
|
||||
void tickintr_establish(void);
|
||||
void tickintr_establish(int, int (*)(void *));
|
||||
/* locore.s */
|
||||
struct fpstate64;
|
||||
void savefpstate(struct fpstate64 *);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: intr.h,v 1.21 2008/03/02 22:01:38 martin Exp $ */
|
||||
/* $NetBSD: intr.h,v 1.22 2008/03/14 15:38:00 nakayama Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1998 The NetBSD Foundation, Inc.
|
||||
|
|
@ -39,7 +39,9 @@
|
|||
#ifndef _SPARC64_INTR_H_
|
||||
#define _SPARC64_INTR_H_
|
||||
|
||||
#ifndef _LOCORE
|
||||
#include <machine/cpuset.h>
|
||||
#endif
|
||||
|
||||
/* XXX - arbitrary numbers; no interpretation is defined yet */
|
||||
#define IPL_NONE 0 /* nothing */
|
||||
|
|
@ -54,13 +56,17 @@
|
|||
#define IPL_PAUSE 13 /* pause cpu */
|
||||
#define IPL_FDSOFT PIL_FDSOFT /* floppy */
|
||||
|
||||
void save_and_clear_fpstate(struct lwp *);
|
||||
#ifndef _LOCORE
|
||||
void fpusave_lwp(struct lwp *, bool);
|
||||
#endif /* _LOCORE */
|
||||
|
||||
#if defined(MULTIPROCESSOR)
|
||||
#ifndef _LOCORE
|
||||
void sparc64_ipi_init (void);
|
||||
int sparc64_ipi_halt_thiscpu (void *);
|
||||
int sparc64_ipi_pause_thiscpu (void *);
|
||||
void sparc64_do_pause(void);
|
||||
void sparc64_ipi_sync_tick (void *);
|
||||
void sparc64_ipi_drop_fpstate (void *);
|
||||
void sparc64_ipi_save_fpstate (void *);
|
||||
void sparc64_ipi_nop (void *);
|
||||
|
|
@ -69,6 +75,16 @@ void mp_pause_cpus (void);
|
|||
void mp_resume_cpus (void);
|
||||
int mp_cpu_is_paused (sparc64_cpuset_t);
|
||||
void mp_resume_cpu(int);
|
||||
#endif /* _LOCORE */
|
||||
|
||||
#define IPI_EVCNT_TLB_PTE 0
|
||||
#define IPI_EVCNT_TLB_CTX 1
|
||||
#define IPI_EVCNT_TLB_ALL 2
|
||||
#define IPI_EVCNT_FPU_SYNCH 3
|
||||
#define IPI_EVCNT_FPU_FLUSH 4
|
||||
#define IPI_EVCNT_NUM 5
|
||||
#define IPI_EVCNT_NAMES { "TLB pte IPI", "TLB ctx IPI", "TLB all IPI", \
|
||||
"FPU synch IPI", "FPU flush IPI" }
|
||||
#endif
|
||||
|
||||
#endif /* _SPARC64_INTR_H_ */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: cpu.c,v 1.69 2008/03/06 09:33:03 nakayama Exp $ */
|
||||
/* $NetBSD: cpu.c,v 1.70 2008/03/14 15:38:00 nakayama Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1996
|
||||
|
|
@ -52,7 +52,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.69 2008/03/06 09:33:03 nakayama Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.70 2008/03/14 15:38:00 nakayama Exp $");
|
||||
|
||||
#include "opt_multiprocessor.h"
|
||||
|
||||
|
|
@ -90,6 +90,10 @@ char machine_arch[] = MACHINE_ARCH; /* from <machine/param.h> */
|
|||
char cpu_model[100]; /* machine model (primary CPU) */
|
||||
extern char machine_model[];
|
||||
|
||||
#ifdef MULTIPROCESSOR
|
||||
static const char *ipi_evcnt_names[IPI_EVCNT_NUM] = IPI_EVCNT_NAMES;
|
||||
#endif
|
||||
|
||||
static void cpu_reset_fpustate(void);
|
||||
|
||||
/* The CPU configuration driver. */
|
||||
|
|
@ -236,6 +240,9 @@ cpu_attach(struct device *parent, struct device *dev, void *aux)
|
|||
mi_cpu_attach(ci);
|
||||
ci->ci_cpcb = (struct pcb *)ci->ci_data.cpu_idlelwp->l_addr;
|
||||
}
|
||||
for (i = 0; i < IPI_EVCNT_NUM; ++i)
|
||||
evcnt_attach_dynamic(&ci->ci_ipi_evcnt[i], EVCNT_TYPE_INTR,
|
||||
NULL, dev->dv_xname, ipi_evcnt_names[i]);
|
||||
#endif
|
||||
evcnt_attach_dynamic(&ci->ci_tick_evcnt, EVCNT_TYPE_INTR, NULL,
|
||||
dev->dv_xname, "timer");
|
||||
|
|
@ -392,6 +399,12 @@ cpu_boot_secondary_processors()
|
|||
if (!CPUSET_HAS(cpus_active, ci->ci_index))
|
||||
printf("cpu%d: startup failed\n", ci->ci_cpuid);
|
||||
}
|
||||
|
||||
/*
|
||||
* Sync %tick register with primary CPU.
|
||||
* No need this actually since we can use counter-timer as timecounter.
|
||||
*/
|
||||
sparc64_broadcast_ipi(sparc64_ipi_sync_tick, tick() + 100, 0);
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -408,7 +421,7 @@ cpu_hatch()
|
|||
cpu_reset_fpustate();
|
||||
curlwp = curcpu()->ci_data.cpu_idlelwp;
|
||||
membar_sync();
|
||||
tickintr_establish();
|
||||
tickintr_establish(14, statintr);
|
||||
spl0();
|
||||
}
|
||||
#endif /* MULTIPROCESSOR */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# $NetBSD: genassym.cf,v 1.53 2008/03/02 15:28:26 nakayama Exp $
|
||||
# $NetBSD: genassym.cf,v 1.54 2008/03/14 15:38:00 nakayama Exp $
|
||||
|
||||
#
|
||||
# Copyright (c) 1997 The NetBSD Foundation, Inc.
|
||||
|
|
@ -178,6 +178,9 @@ define CI_INTRLEV0 offsetof(struct cpu_info, ci_intrlev0)
|
|||
define CI_CTXBUSY offsetof(struct cpu_info, ci_ctxbusy)
|
||||
define CI_TSB_DMMU offsetof(struct cpu_info, ci_tsb_dmmu)
|
||||
define CI_TSB_IMMU offsetof(struct cpu_info, ci_tsb_immu)
|
||||
ifdef MULTIPROCESSOR
|
||||
define CI_IPIEVC offsetof(struct cpu_info, ci_ipi_evcnt[0].ev_count)
|
||||
endif
|
||||
|
||||
# CPU boot arguments structure
|
||||
define CBA_NODE offsetof(struct cpu_bootargs, cb_node)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: ipifuncs.c,v 1.14 2008/03/02 22:01:38 martin Exp $ */
|
||||
/* $NetBSD: ipifuncs.c,v 1.15 2008/03/14 15:38:00 nakayama Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2004 The NetBSD Foundation, Inc.
|
||||
|
|
@ -34,7 +34,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.14 2008/03/02 22:01:38 martin Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.15 2008/03/14 15:38:00 nakayama Exp $");
|
||||
|
||||
#include "opt_ddb.h"
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ __KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.14 2008/03/02 22:01:38 martin Exp $")
|
|||
|
||||
#include <machine/cpu.h>
|
||||
#include <machine/ctlreg.h>
|
||||
#include <machine/pte.h>
|
||||
#include <machine/pmap.h>
|
||||
#include <machine/sparc64.h>
|
||||
|
||||
#define SPARC64_IPI_RETRIES 10000
|
||||
|
|
@ -89,6 +89,8 @@ sparc64_ipi_halt_thiscpu(void *arg)
|
|||
|
||||
printf("cpu%d: shutting down\n", cpu_number());
|
||||
CPUSET_ADD(cpus_halted, cpu_number());
|
||||
if (CPU_IS_PRIMARY(curcpu()))
|
||||
DELAY(1000000); /* XXX - wait for interrupter's halt */
|
||||
prom_stopself();
|
||||
|
||||
return(1);
|
||||
|
|
@ -164,7 +166,8 @@ sparc64_ipi_init()
|
|||
* Send an IPI to all in the list but ourselves.
|
||||
*/
|
||||
void
|
||||
sparc64_multicast_ipi(sparc64_cpuset_t cpuset, ipifunc_t func, uint64_t arg1)
|
||||
sparc64_multicast_ipi(sparc64_cpuset_t cpuset, ipifunc_t func, uint64_t arg1,
|
||||
uint64_t arg2)
|
||||
{
|
||||
struct cpu_info *ci;
|
||||
|
||||
|
|
@ -175,7 +178,7 @@ sparc64_multicast_ipi(sparc64_cpuset_t cpuset, ipifunc_t func, uint64_t arg1)
|
|||
for (ci = cpus; ci != NULL; ci = ci->ci_next) {
|
||||
if (CPUSET_HAS(cpuset, ci->ci_index)) {
|
||||
CPUSET_DEL(cpuset, ci->ci_index);
|
||||
sparc64_send_ipi(ci->ci_cpuid, func, arg1);
|
||||
sparc64_send_ipi(ci->ci_cpuid, func, arg1, arg2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -184,18 +187,18 @@ sparc64_multicast_ipi(sparc64_cpuset_t cpuset, ipifunc_t func, uint64_t arg1)
|
|||
* Broadcast an IPI to all but ourselves.
|
||||
*/
|
||||
void
|
||||
sparc64_broadcast_ipi(ipifunc_t func, uint64_t arg1)
|
||||
sparc64_broadcast_ipi(ipifunc_t func, uint64_t arg1, uint64_t arg2)
|
||||
{
|
||||
|
||||
sparc64_multicast_ipi(CPUSET_EXCEPT(cpus_active, cpu_number()), func,
|
||||
arg1);
|
||||
arg1, arg2);
|
||||
}
|
||||
|
||||
/*
|
||||
* Send an interprocessor interrupt.
|
||||
*/
|
||||
void
|
||||
sparc64_send_ipi(int upaid, ipifunc_t func, uint64_t arg1)
|
||||
sparc64_send_ipi(int upaid, ipifunc_t func, uint64_t arg1, uint64_t arg2)
|
||||
{
|
||||
int i, ik;
|
||||
uint64_t intr_func;
|
||||
|
|
@ -209,11 +212,14 @@ sparc64_send_ipi(int upaid, ipifunc_t func, uint64_t arg1)
|
|||
for (i = 0; i < SPARC64_IPI_RETRIES; i++) {
|
||||
int s = intr_disable();
|
||||
|
||||
stxa(IDDR_0H, ASI_INTERRUPT_DISPATCH, 0);
|
||||
stxa(IDDR_1H, ASI_INTERRUPT_DISPATCH, intr_func);
|
||||
stxa(IDDR_2H, ASI_INTERRUPT_DISPATCH, arg1);
|
||||
stxa(IDDR_0H, ASI_INTERRUPT_DISPATCH, intr_func);
|
||||
stxa(IDDR_1H, ASI_INTERRUPT_DISPATCH, arg1);
|
||||
stxa(IDDR_2H, ASI_INTERRUPT_DISPATCH, arg2);
|
||||
stxa(IDCR(upaid), ASI_INTERRUPT_DISPATCH, 0);
|
||||
membar_sync();
|
||||
/* Workaround for SpitFire erratum #54, from FreeBSD */
|
||||
(void)ldxa(P_DCR_0, ASI_DATAPATH_READ);
|
||||
membar_sync();
|
||||
|
||||
for (ik = 0; ik < 1000000; ik++) {
|
||||
if (ldxa(0, ASR_IDSR) & IDSR_BUSY)
|
||||
|
|
@ -268,7 +274,7 @@ mp_halt_cpus()
|
|||
if (CPUSET_EMPTY(cpuset))
|
||||
return;
|
||||
|
||||
sparc64_multicast_ipi(cpuset, sparc64_ipi_halt, 0);
|
||||
sparc64_multicast_ipi(cpuset, sparc64_ipi_halt, 0, 0);
|
||||
if (sparc64_ipi_wait(&cpus_halted, cpumask))
|
||||
sparc64_ipi_error("halt", cpumask, cpus_halted);
|
||||
}
|
||||
|
|
@ -287,7 +293,7 @@ mp_pause_cpus()
|
|||
if (CPUSET_EMPTY(cpuset))
|
||||
return;
|
||||
|
||||
sparc64_multicast_ipi(cpuset, sparc64_ipi_pause, 0);
|
||||
sparc64_multicast_ipi(cpuset, sparc64_ipi_pause, 0, 0);
|
||||
if (sparc64_ipi_wait(&cpus_paused, cpuset))
|
||||
sparc64_ipi_error("pause", cpus_paused, cpuset);
|
||||
}
|
||||
|
|
@ -331,26 +337,72 @@ mp_cpu_is_paused(sparc64_cpuset_t cpunum)
|
|||
* Flush pte on all active processors.
|
||||
*/
|
||||
void
|
||||
smp_tlb_flush_pte(vaddr_t va, int ctx)
|
||||
smp_tlb_flush_pte(vaddr_t va, pmap_t pm)
|
||||
{
|
||||
sparc64_cpuset_t cpuset;
|
||||
struct cpu_info *ci;
|
||||
int ctx;
|
||||
bool kpm = (pm == pmap_kernel());
|
||||
|
||||
/* Flush our own TLB */
|
||||
sp_tlb_flush_pte(va, ctx);
|
||||
ctx = pm->pm_ctx[cpu_number()];
|
||||
KASSERT(ctx >= 0);
|
||||
if (kpm || ctx > 0)
|
||||
sp_tlb_flush_pte(va, ctx);
|
||||
|
||||
CPUSET_ASSIGN(cpuset, cpus_active);
|
||||
CPUSET_DEL(cpuset, cpu_number());
|
||||
if (CPUSET_EMPTY(cpuset))
|
||||
return;
|
||||
|
||||
/* Flush others */
|
||||
/* sparc64_broadcast_ipi(sparc64_ipi_flush_ctx, ctx); */
|
||||
for (ci = cpus; ci != NULL; ci = ci->ci_next) {
|
||||
if (CPUSET_HAS(cpuset, ci->ci_index)) {
|
||||
CPUSET_DEL(cpuset, ci->ci_index);
|
||||
ctx = pm->pm_ctx[ci->ci_index];
|
||||
KASSERT(ctx >= 0);
|
||||
if (!kpm && ctx == 0)
|
||||
continue;
|
||||
sparc64_send_ipi(ci->ci_cpuid, sparc64_ipi_flush_pte,
|
||||
va, ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush context on all active processors.
|
||||
*/
|
||||
void
|
||||
smp_tlb_flush_ctx(int ctx)
|
||||
smp_tlb_flush_ctx(pmap_t pm)
|
||||
{
|
||||
sparc64_cpuset_t cpuset;
|
||||
struct cpu_info *ci;
|
||||
int ctx;
|
||||
bool kpm = (pm == pmap_kernel());
|
||||
|
||||
/* Flush our own TLB */
|
||||
sp_tlb_flush_ctx(ctx);
|
||||
ctx = pm->pm_ctx[cpu_number()];
|
||||
KASSERT(ctx >= 0);
|
||||
if (kpm || ctx > 0)
|
||||
sp_tlb_flush_ctx(ctx);
|
||||
|
||||
CPUSET_ASSIGN(cpuset, cpus_active);
|
||||
CPUSET_DEL(cpuset, cpu_number());
|
||||
if (CPUSET_EMPTY(cpuset))
|
||||
return;
|
||||
|
||||
/* Flush others */
|
||||
sparc64_broadcast_ipi(sparc64_ipi_flush_ctx, ctx);
|
||||
for (ci = cpus; ci != NULL; ci = ci->ci_next) {
|
||||
if (CPUSET_HAS(cpuset, ci->ci_index)) {
|
||||
CPUSET_DEL(cpuset, ci->ci_index);
|
||||
ctx = pm->pm_ctx[ci->ci_index];
|
||||
KASSERT(ctx >= 0);
|
||||
if (!kpm && ctx == 0)
|
||||
continue;
|
||||
sparc64_send_ipi(ci->ci_cpuid, sparc64_ipi_flush_ctx,
|
||||
ctx, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -363,7 +415,7 @@ smp_tlb_flush_all()
|
|||
sp_tlb_flush_all();
|
||||
|
||||
/* Flush others */
|
||||
sparc64_broadcast_ipi(sparc64_ipi_flush_all, 0);
|
||||
sparc64_broadcast_ipi(sparc64_ipi_flush_all, 0, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore.s,v 1.269 2008/03/02 15:28:26 nakayama Exp $ */
|
||||
/* $NetBSD: locore.s,v 1.270 2008/03/14 15:38:00 nakayama Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1996-2002 Eduardo Horvath
|
||||
|
|
@ -83,6 +83,7 @@
|
|||
#include <machine/frame.h>
|
||||
#include <machine/pte.h>
|
||||
#include <machine/pmap.h>
|
||||
#include <machine/intr.h>
|
||||
#include <machine/asm.h>
|
||||
#include <sys/syscall.h>
|
||||
|
||||
|
|
@ -3623,8 +3624,8 @@ interrupt_vector:
|
|||
stw %g2, [%g1]
|
||||
#endif
|
||||
ldxa [%g0] ASI_IRSR, %g1
|
||||
mov IRDR_0H, %g2
|
||||
ldxa [%g2] ASI_IRDR, %g2 ! Get interrupt number
|
||||
mov IRDR_0H, %g7
|
||||
ldxa [%g7] ASI_IRDR, %g7 ! Get interrupt number
|
||||
membar #Sync
|
||||
|
||||
#if KTR_COMPILE & KTR_INTR
|
||||
|
|
@ -3633,32 +3634,33 @@ interrupt_vector:
|
|||
rdpr %tl, %g5
|
||||
stx %g5, [%g3 + KTR_PARM1]
|
||||
stx %g1, [%g3 + KTR_PARM2]
|
||||
stx %g2, [%g3 + KTR_PARM3]
|
||||
stx %g7, [%g3 + KTR_PARM3]
|
||||
12:
|
||||
#endif
|
||||
|
||||
btst IRSR_BUSY, %g1
|
||||
bz,pn %icc, 3f ! spurious interrupt
|
||||
sllx %g2, PTRSHFT, %g5 ! Calculate entry number
|
||||
sethi %hi(KERNBASE), %g1
|
||||
|
||||
brnz,pt %g2, Lsoftint_regular ! interrupt #0 is a fast cross-call
|
||||
cmp %g2, MAXINTNUM
|
||||
cmp %g7, %g1
|
||||
bl,pt %xcc, Lsoftint_regular ! >= KERNBASE is a fast cross-call
|
||||
cmp %g7, MAXINTNUM
|
||||
|
||||
mov IRDR_1H, %g1
|
||||
ldxa [%g1] ASI_IRDR, %g1 ! Get IPI handler address
|
||||
brz,pn %g1, ret_from_intr_vector
|
||||
mov IRDR_2H, %g2
|
||||
ldxa [%g2] ASI_IRDR, %g2 ! Get IPI handler argument
|
||||
mov IRDR_1H, %g2
|
||||
ldxa [%g2] ASI_IRDR, %g2 ! Get IPI handler argument 1
|
||||
mov IRDR_2H, %g3
|
||||
ldxa [%g3] ASI_IRDR, %g3 ! Get IPI handler argument 2
|
||||
|
||||
stxa %g0, [%g0] ASI_IRSR ! Ack IRQ
|
||||
membar #Sync ! Should not be needed due to retry
|
||||
|
||||
jmpl %g1, %g0
|
||||
jmpl %g7, %g0
|
||||
nop
|
||||
|
||||
Lsoftint_regular:
|
||||
stxa %g0, [%g0] ASI_IRSR ! Ack IRQ
|
||||
membar #Sync ! Should not be needed due to retry
|
||||
sllx %g7, PTRSHFT, %g5 ! Calculate entry number
|
||||
sethi %hi(_C_LABEL(intrlev)), %g3
|
||||
bgeu,pn %xcc, 3f
|
||||
or %g3, %lo(_C_LABEL(intrlev)), %g3
|
||||
|
|
@ -3704,16 +3706,16 @@ ret_from_intr_vector:
|
|||
|
||||
3:
|
||||
#ifdef NOT_DEBUG
|
||||
set _C_LABEL(intrdebug), %g7
|
||||
ld [%g7], %g7
|
||||
btst INTRDEBUG_SPUR, %g7
|
||||
set _C_LABEL(intrdebug), %g6
|
||||
ld [%g6], %g6
|
||||
btst INTRDEBUG_SPUR, %g6
|
||||
bz,pt %icc, 97f
|
||||
nop
|
||||
#endif
|
||||
#if 1
|
||||
STACKFRAME(-CC64FSZ) ! Get a clean register window
|
||||
LOAD_ASCIZ(%o0, "interrupt_vector: spurious vector %lx at pil %d\r\n")
|
||||
mov %g2, %o1
|
||||
mov %g7, %o1
|
||||
GLOBTOLOC
|
||||
clr %g4
|
||||
call prom_printf
|
||||
|
|
@ -3756,13 +3758,42 @@ sparc64_ipi_pause_trap_point:
|
|||
ba,a ret_from_intr_vector
|
||||
nop
|
||||
|
||||
/*
|
||||
* IPI handler to sync %tick register.
|
||||
* void sparc64_ipi_sync_tick(void *);
|
||||
*
|
||||
* On entry:
|
||||
* %g2 = %tick of master CPU
|
||||
*/
|
||||
ENTRY(sparc64_ipi_sync_tick)
|
||||
rdpr %pstate, %g1
|
||||
andn %g1, PSTATE_IE, %g7 ! disable interrupts
|
||||
wrpr %g7, 0, %pstate
|
||||
rdpr %tick, %g3 ! read old tick
|
||||
rd TICK_CMPR, %g5
|
||||
sub %g2, %g3, %g3 ! delta btw old and new
|
||||
add %g5, %g3, %g5 ! add delta to TICK_CMPR
|
||||
wr %g5, TICK_CMPR
|
||||
wrpr %g2, 0, %tick ! write new tick
|
||||
ba ret_from_intr_vector
|
||||
wrpr %g1, %pstate ! restore interrupts
|
||||
|
||||
/*
|
||||
* Increment IPI event counter, defined in machine/{cpu,intr}.h.
|
||||
*/
|
||||
#define IPIEVC_INC(n,r1,r2) \
|
||||
sethi %hi(CPUINFO_VA+CI_IPIEVC+EVC_SIZE*n), r2; \
|
||||
ldx [r2 + %lo(CPUINFO_VA+CI_IPIEVC+EVC_SIZE*n)], r1; \
|
||||
inc r1; \
|
||||
stx r1, [r2 + %lo(CPUINFO_VA+CI_IPIEVC+EVC_SIZE*n)]
|
||||
|
||||
/*
|
||||
* IPI handler to flush single pte.
|
||||
* void sparc64_ipi_flush_pte(void *);
|
||||
*
|
||||
* On Entry:
|
||||
*
|
||||
* %g2 - pointer to 'ipi_tlb_args' structure
|
||||
* On entry:
|
||||
* %g2 = vaddr_t va
|
||||
* %g3 = int ctx
|
||||
*/
|
||||
ENTRY(sparc64_ipi_flush_pte)
|
||||
#if KTR_COMPILE & KTR_PMAP
|
||||
|
|
@ -3770,24 +3801,28 @@ ENTRY(sparc64_ipi_flush_pte)
|
|||
%g1, %g3, %g4, 10, 11, 12)
|
||||
12:
|
||||
#endif
|
||||
#if 0
|
||||
! save %o0 - %o5
|
||||
mov %o0, %g1
|
||||
mov %o1, %g3
|
||||
mov %o2, %g4
|
||||
mov %o3, %g5
|
||||
mov %o4, %g6
|
||||
mov %o5, %g7
|
||||
LDPTR [%g2 + ITA_VADDR], %o0
|
||||
call sp_tlb_flush_pte
|
||||
ld [%g2 + ITA_CTX], %o1
|
||||
! restore %o0 - %o5
|
||||
mov %g1, %o0
|
||||
mov %g3, %o1
|
||||
mov %g4, %o2
|
||||
mov %g5, %o3
|
||||
mov %g6, %o4
|
||||
mov %g7, %o5
|
||||
#ifdef SPITFIRE
|
||||
mov CTX_SECONDARY, %g5
|
||||
andn %g2, 0xfff, %g2 ! drop unused va bits
|
||||
ldxa [%g5] ASI_DMMU, %g6 ! Save secondary context
|
||||
sethi %hi(KERNBASE), %g7
|
||||
membar #LoadStore
|
||||
stxa %g3, [%g5] ASI_DMMU ! Insert context to demap
|
||||
membar #Sync
|
||||
or %g2, DEMAP_PAGE_SECONDARY, %g2 ! Demap page from secondary context only
|
||||
stxa %g2, [%g2] ASI_DMMU_DEMAP ! Do the demap
|
||||
stxa %g2, [%g2] ASI_IMMU_DEMAP ! to both TLBs
|
||||
#ifdef _LP64
|
||||
srl %g2, 0, %g2 ! and make sure it's both 32- and 64-bit entries
|
||||
stxa %g2, [%g2] ASI_DMMU_DEMAP ! Do the demap
|
||||
stxa %g2, [%g2] ASI_IMMU_DEMAP ! Do the demap
|
||||
#endif
|
||||
flush %g7
|
||||
stxa %g6, [%g5] ASI_DMMU ! Restore secondary context
|
||||
membar #Sync
|
||||
IPIEVC_INC(IPI_EVCNT_TLB_PTE,%g2,%g3)
|
||||
#else
|
||||
! Not yet
|
||||
#endif
|
||||
|
||||
ba,a ret_from_intr_vector
|
||||
|
|
@ -3797,9 +3832,8 @@ ENTRY(sparc64_ipi_flush_pte)
|
|||
* IPI handler to flush single context.
|
||||
* void sparc64_ipi_flush_ctx(void *);
|
||||
*
|
||||
* On Entry:
|
||||
*
|
||||
* %g2 - pointer to 'ipi_tlb_args' structure
|
||||
* On entry:
|
||||
* %g2 = int ctx
|
||||
*/
|
||||
ENTRY(sparc64_ipi_flush_ctx)
|
||||
#if KTR_COMPILE & KTR_PMAP
|
||||
|
|
@ -3807,23 +3841,22 @@ ENTRY(sparc64_ipi_flush_ctx)
|
|||
%g1, %g3, %g4, 10, 11, 12)
|
||||
12:
|
||||
#endif
|
||||
#if 0
|
||||
! save %o0 - %o5
|
||||
mov %o0, %g1
|
||||
mov %o1, %g3
|
||||
mov %o2, %g4
|
||||
mov %o3, %g5
|
||||
mov %o4, %g6
|
||||
mov %o5, %g7
|
||||
call sp_tlb_flush_ctx
|
||||
ld [%g2 + ITA_CTX], %o0
|
||||
! restore %o0 - %o5
|
||||
mov %g1, %o0
|
||||
mov %g3, %o1
|
||||
mov %g4, %o2
|
||||
mov %g5, %o3
|
||||
mov %g6, %o4
|
||||
mov %g7, %o5
|
||||
#ifdef SPITFIRE
|
||||
mov CTX_SECONDARY, %g5
|
||||
ldxa [%g5] ASI_DMMU, %g6 ! Save secondary context
|
||||
sethi %hi(KERNBASE), %g7
|
||||
membar #LoadStore
|
||||
stxa %g2, [%g5] ASI_DMMU ! Insert context to demap
|
||||
set DEMAP_CTX_SECONDARY, %g3
|
||||
membar #Sync
|
||||
stxa %g3, [%g3] ASI_DMMU_DEMAP ! Do the demap
|
||||
stxa %g3, [%g3] ASI_IMMU_DEMAP ! Do the demap
|
||||
flush %g7
|
||||
stxa %g6, [%g5] ASI_DMMU ! Restore secondary context
|
||||
membar #Sync
|
||||
IPIEVC_INC(IPI_EVCNT_TLB_CTX,%g2,%g3)
|
||||
#else
|
||||
! Not yet
|
||||
#endif
|
||||
|
||||
ba,a ret_from_intr_vector
|
||||
|
|
@ -5355,7 +5388,8 @@ ENTRY(openfirmware_exit)
|
|||
flushw ! Flush register windows
|
||||
|
||||
wrpr %g0, PIL_HIGH, %pil ! Disable interrupts
|
||||
set romtba, %l5
|
||||
sethi %hi(romtba), %l5
|
||||
LDPTR [%l5 + %lo(romtba)], %l5
|
||||
wrpr %l5, 0, %tba ! restore the ofw trap table
|
||||
|
||||
/* Arrange locked kernel stack as PROM stack */
|
||||
|
|
@ -5368,8 +5402,8 @@ ENTRY(openfirmware_exit)
|
|||
mov %l5, %sp
|
||||
flushw
|
||||
|
||||
set romp, %l6
|
||||
LDPTR [%l6], %l6
|
||||
sethi %hi(romp), %l6
|
||||
LDPTR [%l6 + %lo(romp)], %l6
|
||||
|
||||
mov CTX_PRIMARY, %l3 ! set context 0
|
||||
stxa %g0, [%l3] ASI_DMMU
|
||||
|
|
@ -5417,9 +5451,14 @@ ENTRY(sp_tlb_flush_pte)
|
|||
2:
|
||||
#endif
|
||||
#ifdef SPITFIRE
|
||||
#ifdef MULTIPROCESSOR
|
||||
rdpr %pstate, %o3
|
||||
andn %o3, PSTATE_IE, %o4 ! disable interrupts
|
||||
wrpr %o4, 0, %pstate
|
||||
#endif
|
||||
mov CTX_SECONDARY, %o2
|
||||
andn %o0, 0xfff, %o0 ! drop unused va bits
|
||||
ldxa [%o2] ASI_DMMU, %g1 ! Save secondary context
|
||||
ldxa [%o2] ASI_DMMU, %o5 ! Save secondary context
|
||||
sethi %hi(KERNBASE), %o4
|
||||
membar #LoadStore
|
||||
stxa %o1, [%o2] ASI_DMMU ! Insert context to demap
|
||||
|
|
@ -5427,14 +5466,20 @@ ENTRY(sp_tlb_flush_pte)
|
|||
or %o0, DEMAP_PAGE_SECONDARY, %o0 ! Demap page from secondary context only
|
||||
stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap
|
||||
stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs
|
||||
#ifdef _LP64
|
||||
srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries
|
||||
stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap
|
||||
stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap
|
||||
#endif
|
||||
flush %o4
|
||||
stxa %g1, [%o2] ASI_DMMU ! Restore secondary context
|
||||
stxa %o5, [%o2] ASI_DMMU ! Restore secondary context
|
||||
membar #Sync
|
||||
retl
|
||||
#ifdef MULTIPROCESSOR
|
||||
wrpr %o3, %pstate ! restore interrupts
|
||||
#else
|
||||
nop
|
||||
#endif
|
||||
#else
|
||||
!!
|
||||
!! Cheetahs do not support flushing the IMMU from secondary context
|
||||
|
|
@ -5506,6 +5551,11 @@ ENTRY(sp_tlb_flush_ctx)
|
|||
2:
|
||||
#endif
|
||||
#ifdef SPITFIRE
|
||||
#ifdef MULTIPROCESSOR
|
||||
rdpr %pstate, %o3
|
||||
andn %o3, PSTATE_IE, %o4 ! disable interrupts
|
||||
wrpr %o4, 0, %pstate
|
||||
#endif
|
||||
mov CTX_SECONDARY, %o2
|
||||
ldxa [%o2] ASI_DMMU, %o1 ! Save secondary context
|
||||
sethi %hi(KERNBASE), %o4
|
||||
|
|
@ -5515,11 +5565,15 @@ ENTRY(sp_tlb_flush_ctx)
|
|||
membar #Sync
|
||||
stxa %o5, [%o5] ASI_DMMU_DEMAP ! Do the demap
|
||||
stxa %o5, [%o5] ASI_IMMU_DEMAP ! Do the demap
|
||||
membar #Sync
|
||||
stxa %o1, [%o2] ASI_DMMU ! Restore secondary asi
|
||||
flush %o4
|
||||
stxa %o1, [%o2] ASI_DMMU ! Restore secondary context
|
||||
membar #Sync
|
||||
retl
|
||||
#ifdef MULTIPROCESSOR
|
||||
wrpr %o3, %pstate ! restore interrupts
|
||||
#else
|
||||
nop
|
||||
#endif
|
||||
#else
|
||||
rdpr %tl, %o3
|
||||
mov CTX_PRIMARY, %o2
|
||||
|
|
@ -5554,7 +5608,6 @@ ENTRY(sp_tlb_flush_ctx)
|
|||
.align 8
|
||||
ENTRY(sp_tlb_flush_all)
|
||||
#ifdef SPITFIRE
|
||||
save %sp, -CC64FSZ, %sp
|
||||
rdpr %pstate, %o3
|
||||
andn %o3, PSTATE_IE, %o4 ! disable interrupts
|
||||
wrpr %o4, 0, %pstate
|
||||
|
|
@ -5616,11 +5669,8 @@ ENTRY(sp_tlb_flush_all)
|
|||
sethi %hi(KERNBASE), %o4
|
||||
membar #Sync
|
||||
flush %o4
|
||||
! retl
|
||||
retl
|
||||
wrpr %o3, %pstate
|
||||
|
||||
ret
|
||||
restore
|
||||
#else
|
||||
WRITEME
|
||||
#endif
|
||||
|
|
@ -9459,23 +9509,136 @@ Lkcerr:
|
|||
NOTREACHED
|
||||
|
||||
#ifdef MULTIPROCESSOR
|
||||
/*
|
||||
* IPI handler to store the current FPU state.
|
||||
* void sparc64_ipi_save_fpstate(void *);
|
||||
*/
|
||||
ENTRY(sparc64_ipi_save_fpstate)
|
||||
save %sp, -CC64FSZ, %sp
|
||||
sethi %hi(FPLWP), %o0
|
||||
LDPTR [%o0 + %lo(FPLWP)], %o0
|
||||
call savefpstate
|
||||
LDPTR [%o0 + L_FPSTATE], %o0
|
||||
sethi %hi(FPLWP), %o0
|
||||
STPTR %g0, [%o0 + %lo(FPLWP)] ! fplwp = NULL
|
||||
ba ret_from_intr_vector
|
||||
restore
|
||||
mov CTX_SECONDARY, %g5
|
||||
ldxa [%g5] ASI_DMMU, %g6
|
||||
membar #LoadStore
|
||||
stxa %g0, [%g5] ASI_DMMU
|
||||
membar #Sync
|
||||
|
||||
sethi %hi(FPLWP), %g1
|
||||
LDPTR [%g1 + %lo(FPLWP)], %g3
|
||||
LDPTR [%g3 + L_FPSTATE], %g3
|
||||
|
||||
rdpr %pstate, %g2 ! enable FP before we begin
|
||||
rd %fprs, %g5
|
||||
wr %g0, FPRS_FEF, %fprs
|
||||
or %g2, PSTATE_PEF, %g2
|
||||
wrpr %g2, 0, %pstate
|
||||
|
||||
stx %fsr, [%g3 + FS_FSR] ! f->fs_fsr = getfsr();
|
||||
|
||||
rd %gsr, %g2 ! Save %gsr
|
||||
st %g2, [%g3 + FS_GSR]
|
||||
|
||||
add %g3, FS_REGS, %g3
|
||||
btst BLOCK_ALIGN, %g3 ! Needs to be re-executed
|
||||
bnz,pn %icc, 3f ! Check alignment
|
||||
st %g0, [%g3 + FS_QSIZE - FS_REGS] ! f->fs_qsize = 0;
|
||||
btst FPRS_DL, %g5 ! Lower FPU clean?
|
||||
bz,a,pt %icc, 1f ! Then skip it
|
||||
add %g3, 128, %g3 ! Skip a block
|
||||
membar #Sync
|
||||
stda %f0, [%g3] ASI_BLK_S ! f->fs_f0 = etc;
|
||||
inc BLOCK_SIZE, %g3
|
||||
stda %f16, [%g3] ASI_BLK_S
|
||||
inc BLOCK_SIZE, %g3
|
||||
1:
|
||||
btst FPRS_DU, %g5 ! Upper FPU clean?
|
||||
bz,pt %icc, 2f ! Then skip it
|
||||
nop
|
||||
|
||||
membar #Sync
|
||||
stda %f32, [%g3] ASI_BLK_S
|
||||
inc BLOCK_SIZE, %g3
|
||||
stda %f48, [%g3] ASI_BLK_S
|
||||
2:
|
||||
membar #Sync ! Finish operation so we can
|
||||
wr %g0, FPRS_FEF, %fprs ! Mark FPU clean
|
||||
|
||||
mov CTX_SECONDARY, %g5
|
||||
STPTR %g0, [%g1 + %lo(FPLWP)] ! fplwp = NULL
|
||||
stxa %g6, [%g5] ASI_DMMU
|
||||
membar #Sync
|
||||
IPIEVC_INC(IPI_EVCNT_FPU_SYNCH,%g2,%g3)
|
||||
ba,a ret_from_intr_vector
|
||||
nop
|
||||
|
||||
3:
|
||||
#ifdef DIAGONSTIC
|
||||
btst 7, %g3 ! 32-bit aligned!?!?
|
||||
bnz,pn %icc, 6f
|
||||
#endif
|
||||
btst FPRS_DL, %g5 ! Lower FPU clean?
|
||||
bz,a,pt %icc, 4f ! Then skip it
|
||||
add %g3, 128, %g3
|
||||
|
||||
membar #Sync
|
||||
std %f0, [%g3 + FS_REGS + (4*0)] ! f->fs_f0 = etc;
|
||||
std %f2, [%g3 + FS_REGS + (4*2)]
|
||||
std %f4, [%g3 + FS_REGS + (4*4)]
|
||||
std %f6, [%g3 + FS_REGS + (4*6)]
|
||||
std %f8, [%g3 + FS_REGS + (4*8)]
|
||||
std %f10, [%g3 + FS_REGS + (4*10)]
|
||||
std %f12, [%g3 + FS_REGS + (4*12)]
|
||||
std %f14, [%g3 + FS_REGS + (4*14)]
|
||||
std %f16, [%g3 + FS_REGS + (4*16)]
|
||||
std %f18, [%g3 + FS_REGS + (4*18)]
|
||||
std %f20, [%g3 + FS_REGS + (4*20)]
|
||||
std %f22, [%g3 + FS_REGS + (4*22)]
|
||||
std %f24, [%g3 + FS_REGS + (4*24)]
|
||||
std %f26, [%g3 + FS_REGS + (4*26)]
|
||||
std %f28, [%g3 + FS_REGS + (4*28)]
|
||||
std %f30, [%g3 + FS_REGS + (4*30)]
|
||||
4:
|
||||
btst FPRS_DU, %g5 ! Upper FPU clean?
|
||||
bz,pt %icc, 2b ! Then skip it
|
||||
nop
|
||||
|
||||
membar #Sync
|
||||
std %f32, [%g3 + FS_REGS + (4*32)]
|
||||
std %f34, [%g3 + FS_REGS + (4*34)]
|
||||
std %f36, [%g3 + FS_REGS + (4*36)]
|
||||
std %f38, [%g3 + FS_REGS + (4*38)]
|
||||
std %f40, [%g3 + FS_REGS + (4*40)]
|
||||
std %f42, [%g3 + FS_REGS + (4*42)]
|
||||
std %f44, [%g3 + FS_REGS + (4*44)]
|
||||
std %f46, [%g3 + FS_REGS + (4*46)]
|
||||
std %f48, [%g3 + FS_REGS + (4*48)]
|
||||
std %f50, [%g3 + FS_REGS + (4*50)]
|
||||
std %f52, [%g3 + FS_REGS + (4*52)]
|
||||
std %f54, [%g3 + FS_REGS + (4*54)]
|
||||
std %f56, [%g3 + FS_REGS + (4*56)]
|
||||
std %f58, [%g3 + FS_REGS + (4*58)]
|
||||
std %f60, [%g3 + FS_REGS + (4*60)]
|
||||
ba 2b
|
||||
std %f62, [%g3 + FS_REGS + (4*62)]
|
||||
|
||||
!!
|
||||
!! Damn thing is *NOT* aligned on a 64-byte boundary
|
||||
!!
|
||||
6:
|
||||
wr %g0, FPRS_FEF, %fprs
|
||||
ta 1
|
||||
nop
|
||||
ba,a ret_from_intr_vector
|
||||
nop
|
||||
|
||||
/*
|
||||
* IPI handler to drop the current FPU state.
|
||||
* void sparc64_ipi_drop_fpstate(void *);
|
||||
*/
|
||||
ENTRY(sparc64_ipi_drop_fpstate)
|
||||
rdpr %pstate, %g1
|
||||
wr %g0, FPRS_FEF, %fprs
|
||||
or %g1, PSTATE_PEF, %g1
|
||||
wrpr %g1, 0, %pstate
|
||||
sethi %hi(FPLWP), %g1
|
||||
IPIEVC_INC(IPI_EVCNT_FPU_FLUSH,%g2,%g3)
|
||||
ba ret_from_intr_vector
|
||||
STPTR %g0, [%g1 + %lo(FPLWP)] ! fplwp = NULL
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue