- Centralize per-CPU pmap initialization into a new pmap_init_cpu()
function. Call in from pmap_bootstrap() for the boot CPU, and from cpu_hatch() for secondaary CPUs. - Eliminiate the dedicated I-stream memory barrier IPI; handle it all from the TLB shootdown IPI. Const poison, and add some additional memory barriers and a TBIA to the PAUSE IPI. - Completly rewrite TLB management in the alpha pmap module, borrowing somoe ideas from the x86 pmap and adapting them to the alpha environment. See the comments for theory of operation. Add a bunch of stats that can be reported (disabled by default). - Add some additional symbol decorations to improve cache behavior on MP systems. Ensure coherency unit alignment for several structures in the pmap module. Use hashed locks for pmap structures. - Start out all new processes on the kernel page tables until their first trip though pmap_activate() to avoid the potential of polluting the current ASN in TLB with cross-process mappings.
This commit is contained in:
thorpej
parent
f52782ac02
commit
13c5b655f2
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: cpu.c,v 1.98 2020/08/07 14:20:08 fcambus Exp $ */
|
||||
/* $NetBSD: cpu.c,v 1.99 2020/08/29 20:06:59 thorpej Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
|
||||
@ -59,7 +59,7 @@
|
||||
|
||||
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
|
||||
|
||||
__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.98 2020/08/07 14:20:08 fcambus Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.99 2020/08/29 20:06:59 thorpej Exp $");
|
||||
|
||||
#include "opt_ddb.h"
|
||||
#include "opt_multiprocessor.h"
|
||||
@ -549,8 +549,8 @@ cpu_hatch(struct cpu_info *ci)
|
||||
u_long cpu_id = cpu_number();
|
||||
u_long cpumask = (1UL << cpu_id);
|
||||
|
||||
/* Mark the kernel pmap active on this processor. */
|
||||
atomic_or_ulong(&pmap_kernel()->pm_cpus, cpumask);
|
||||
/* pmap initialization for this processor. */
|
||||
pmap_init_cpu(ci);
|
||||
|
||||
/* Initialize trap vectors for this processor. */
|
||||
trap_init();
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: ipifuncs.c,v 1.51 2020/08/15 16:09:07 thorpej Exp $ */
|
||||
/* $NetBSD: ipifuncs.c,v 1.52 2020/08/29 20:06:59 thorpej Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
|
||||
@ -32,7 +32,7 @@
|
||||
|
||||
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
|
||||
|
||||
__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.51 2020/08/15 16:09:07 thorpej Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.52 2020/08/29 20:06:59 thorpej Exp $");
|
||||
|
||||
/*
|
||||
* Interprocessor interrupt handlers.
|
||||
@ -62,21 +62,15 @@ typedef void (*ipifunc_t)(struct cpu_info *, struct trapframe *);
|
||||
|
||||
static void alpha_ipi_halt(struct cpu_info *, struct trapframe *);
|
||||
static void alpha_ipi_microset(struct cpu_info *, struct trapframe *);
|
||||
static void alpha_ipi_imb(struct cpu_info *, struct trapframe *);
|
||||
static void alpha_ipi_ast(struct cpu_info *, struct trapframe *);
|
||||
static void alpha_ipi_pause(struct cpu_info *, struct trapframe *);
|
||||
static void alpha_ipi_xcall(struct cpu_info *, struct trapframe *);
|
||||
static void alpha_ipi_generic(struct cpu_info *, struct trapframe *);
|
||||
|
||||
/*
|
||||
* NOTE: This table must be kept in order with the bit definitions
|
||||
* in <machine/intr.h>.
|
||||
*/
|
||||
const ipifunc_t ipifuncs[ALPHA_NIPIS] = {
|
||||
[ilog2(ALPHA_IPI_HALT)] = alpha_ipi_halt,
|
||||
[ilog2(ALPHA_IPI_MICROSET)] = alpha_ipi_microset,
|
||||
[ilog2(ALPHA_IPI_SHOOTDOWN)] = pmap_do_tlb_shootdown,
|
||||
[ilog2(ALPHA_IPI_IMB)] = alpha_ipi_imb,
|
||||
[ilog2(ALPHA_IPI_SHOOTDOWN)] = pmap_tlb_shootdown_ipi,
|
||||
[ilog2(ALPHA_IPI_AST)] = alpha_ipi_ast,
|
||||
[ilog2(ALPHA_IPI_PAUSE)] = alpha_ipi_pause,
|
||||
[ilog2(ALPHA_IPI_XCALL)] = alpha_ipi_xcall,
|
||||
@ -87,7 +81,6 @@ const char * const ipinames[ALPHA_NIPIS] = {
|
||||
[ilog2(ALPHA_IPI_HALT)] = "halt ipi",
|
||||
[ilog2(ALPHA_IPI_MICROSET)] = "microset ipi",
|
||||
[ilog2(ALPHA_IPI_SHOOTDOWN)] = "shootdown ipi",
|
||||
[ilog2(ALPHA_IPI_IMB)] = "imb ipi",
|
||||
[ilog2(ALPHA_IPI_AST)] = "ast ipi",
|
||||
[ilog2(ALPHA_IPI_PAUSE)] = "pause ipi",
|
||||
[ilog2(ALPHA_IPI_XCALL)] = "xcall ipi",
|
||||
@ -156,7 +149,7 @@ alpha_ipi_process(struct cpu_info *ci, struct trapframe *framep)
|
||||
* Send an interprocessor interrupt.
|
||||
*/
|
||||
void
|
||||
alpha_send_ipi(u_long cpu_id, u_long ipimask)
|
||||
alpha_send_ipi(u_long const cpu_id, u_long const ipimask)
|
||||
{
|
||||
|
||||
KASSERT(cpu_id < hwrpb->rpb_pcs_cnt);
|
||||
@ -171,14 +164,13 @@ alpha_send_ipi(u_long cpu_id, u_long ipimask)
|
||||
* Broadcast an IPI to all but ourselves.
|
||||
*/
|
||||
void
|
||||
alpha_broadcast_ipi(u_long ipimask)
|
||||
alpha_broadcast_ipi(u_long const ipimask)
|
||||
{
|
||||
struct cpu_info *ci;
|
||||
CPU_INFO_ITERATOR cii;
|
||||
u_long cpu_id = cpu_number();
|
||||
u_long cpumask;
|
||||
|
||||
cpumask = cpus_running & ~(1UL << cpu_id);
|
||||
const u_long cpu_id = cpu_number();
|
||||
const u_long cpumask = cpus_running & ~(1UL << cpu_id);
|
||||
|
||||
for (CPU_INFO_FOREACH(cii, ci)) {
|
||||
if ((cpumask & (1UL << ci->ci_cpuid)) == 0)
|
||||
@ -191,7 +183,7 @@ alpha_broadcast_ipi(u_long ipimask)
|
||||
* Send an IPI to all in the list but ourselves.
|
||||
*/
|
||||
void
|
||||
alpha_multicast_ipi(u_long cpumask, u_long ipimask)
|
||||
alpha_multicast_ipi(u_long cpumask, u_long const ipimask)
|
||||
{
|
||||
struct cpu_info *ci;
|
||||
CPU_INFO_ITERATOR cii;
|
||||
@ -209,10 +201,11 @@ alpha_multicast_ipi(u_long cpumask, u_long ipimask)
|
||||
}
|
||||
|
||||
static void
|
||||
alpha_ipi_halt(struct cpu_info *ci, struct trapframe *framep)
|
||||
alpha_ipi_halt(struct cpu_info * const ci,
|
||||
struct trapframe * const framep __unused)
|
||||
{
|
||||
u_long cpu_id = ci->ci_cpuid;
|
||||
u_long wait_mask = (1UL << cpu_id);
|
||||
const u_long cpu_id = ci->ci_cpuid;
|
||||
const u_long wait_mask = (1UL << cpu_id);
|
||||
|
||||
/* Disable interrupts. */
|
||||
(void) splhigh();
|
||||
@ -242,21 +235,16 @@ alpha_ipi_halt(struct cpu_info *ci, struct trapframe *framep)
|
||||
}
|
||||
|
||||
static void
|
||||
alpha_ipi_microset(struct cpu_info *ci, struct trapframe *framep)
|
||||
alpha_ipi_microset(struct cpu_info * const ci,
|
||||
struct trapframe * const framep __unused)
|
||||
{
|
||||
|
||||
cc_calibrate_cpu(ci);
|
||||
}
|
||||
|
||||
static void
|
||||
alpha_ipi_imb(struct cpu_info *ci, struct trapframe *framep)
|
||||
{
|
||||
|
||||
alpha_pal_imb();
|
||||
}
|
||||
|
||||
static void
|
||||
alpha_ipi_ast(struct cpu_info *ci, struct trapframe *framep)
|
||||
alpha_ipi_ast(struct cpu_info * const ci,
|
||||
struct trapframe * const framep __unused)
|
||||
{
|
||||
|
||||
if (ci->ci_onproc != ci->ci_data.cpu_idlelwp)
|
||||
@ -264,16 +252,16 @@ alpha_ipi_ast(struct cpu_info *ci, struct trapframe *framep)
|
||||
}
|
||||
|
||||
static void
|
||||
alpha_ipi_pause(struct cpu_info *ci, struct trapframe *framep)
|
||||
alpha_ipi_pause(struct cpu_info * const ci, struct trapframe * const framep)
|
||||
{
|
||||
u_long cpumask = (1UL << ci->ci_cpuid);
|
||||
const u_long cpumask = (1UL << ci->ci_cpuid);
|
||||
int s;
|
||||
|
||||
s = splhigh();
|
||||
|
||||
/* Point debuggers at our trapframe for register state. */
|
||||
ci->ci_db_regs = framep;
|
||||
|
||||
alpha_wmb();
|
||||
atomic_or_ulong(&ci->ci_flags, CPUF_PAUSED);
|
||||
|
||||
/* Spin with interrupts disabled until we're resumed. */
|
||||
@ -282,12 +270,13 @@ alpha_ipi_pause(struct cpu_info *ci, struct trapframe *framep)
|
||||
} while (cpus_paused & cpumask);
|
||||
|
||||
atomic_and_ulong(&ci->ci_flags, ~CPUF_PAUSED);
|
||||
|
||||
alpha_wmb();
|
||||
ci->ci_db_regs = NULL;
|
||||
|
||||
splx(s);
|
||||
|
||||
/* Do an IMB on the way out, in case the kernel text was changed. */
|
||||
/* Do a TBIA+IMB on the way out, in case things have changed. */
|
||||
ALPHA_TBIA();
|
||||
alpha_pal_imb();
|
||||
}
|
||||
|
||||
@ -296,13 +285,14 @@ alpha_ipi_pause(struct cpu_info *ci, struct trapframe *framep)
|
||||
*/
|
||||
|
||||
static void
|
||||
alpha_ipi_xcall(struct cpu_info *ci, struct trapframe *framep)
|
||||
alpha_ipi_xcall(struct cpu_info * const ci __unused,
|
||||
struct trapframe * const framep __unused)
|
||||
{
|
||||
xc_ipi_handler();
|
||||
}
|
||||
|
||||
void
|
||||
xc_send_ipi(struct cpu_info *ci)
|
||||
xc_send_ipi(struct cpu_info * const ci)
|
||||
{
|
||||
KASSERT(kpreempt_disabled());
|
||||
KASSERT(curcpu() != ci);
|
||||
@ -317,13 +307,14 @@ xc_send_ipi(struct cpu_info *ci)
|
||||
}
|
||||
|
||||
static void
|
||||
alpha_ipi_generic(struct cpu_info *ci, struct trapframe *framep)
|
||||
alpha_ipi_generic(struct cpu_info * const ci __unused,
|
||||
struct trapframe * const framep __unused)
|
||||
{
|
||||
ipi_cpu_handler();
|
||||
}
|
||||
|
||||
void
|
||||
cpu_ipi(struct cpu_info *ci)
|
||||
cpu_ipi(struct cpu_info * const ci)
|
||||
{
|
||||
KASSERT(kpreempt_disabled());
|
||||
KASSERT(curcpu() != ci);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: vm_machdep.c,v 1.115 2020/08/16 18:05:52 thorpej Exp $ */
|
||||
/* $NetBSD: vm_machdep.c,v 1.116 2020/08/29 20:06:59 thorpej Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
|
||||
@ -29,7 +29,7 @@
|
||||
|
||||
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
|
||||
|
||||
__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.115 2020/08/16 18:05:52 thorpej Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.116 2020/08/29 20:06:59 thorpej Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
@ -109,14 +109,12 @@ cpu_lwp_fork(struct lwp *l1, struct lwp *l2, void *stack, size_t stacksize,
|
||||
pcb2->pcb_hw.apcb_usp = alpha_pal_rdusp();
|
||||
|
||||
/*
|
||||
* Put l2's lev1map into its PTBR so that it will be on its
|
||||
* own page tables as the SWPCTX to its PCB is made. ASN
|
||||
* doesn't matter at this point; that will be handled on l2's
|
||||
* first pmap_activate() call.
|
||||
* Put l2 on the kernel's page tables until its first trip
|
||||
* through pmap_activate().
|
||||
*/
|
||||
pmap_t const pmap2 = l2->l_proc->p_vmspace->vm_map.pmap;
|
||||
pcb2->pcb_hw.apcb_ptbr =
|
||||
ALPHA_K0SEG_TO_PHYS((vaddr_t)pmap2->pm_lev1map) >> PGSHIFT;
|
||||
ALPHA_K0SEG_TO_PHYS((vaddr_t)pmap_kernel()->pm_lev1map) >> PGSHIFT;
|
||||
pcb2->pcb_hw.apcb_asn = PMAP_ASN_KERNEL;
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
/*
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: cpu.h,v 1.88 2020/08/29 19:06:33 thorpej Exp $ */
|
||||
/* $NetBSD: cpu.h,v 1.89 2020/08/29 20:07:00 thorpej Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc.
|
||||
@ -128,11 +128,9 @@ struct cpu_info {
|
||||
struct trapframe *ci_db_regs; /* registers for debuggers */
|
||||
uint64_t ci_pcc_freq; /* cpu cycles/second */
|
||||
|
||||
#define CPU_INFO_PMAP_DATA_NQWORDS 16
|
||||
|
||||
struct pmap *ci_pmap; /* currently-activated pmap */
|
||||
/* data used by pmap module */
|
||||
u_long ci_pmap_data[CPU_INFO_PMAP_DATA_NQWORDS];
|
||||
u_int ci_next_asn; /* next ASN to assign */
|
||||
u_long ci_asn_gen; /* current ASN generation */
|
||||
|
||||
#if defined(MULTIPROCESSOR)
|
||||
volatile u_long ci_flags; /* flags; see below */
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: intr.h,v 1.72 2017/01/14 00:35:37 christos Exp $ */
|
||||
/* $NetBSD: intr.h,v 1.73 2020/08/29 20:07:00 thorpej Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2000, 2001, 2002 The NetBSD Foundation, Inc.
|
||||
@ -160,13 +160,12 @@ _splraise(int s)
|
||||
#define ALPHA_IPI_HALT (1UL << 0)
|
||||
#define ALPHA_IPI_MICROSET (1UL << 1)
|
||||
#define ALPHA_IPI_SHOOTDOWN (1UL << 2)
|
||||
#define ALPHA_IPI_IMB (1UL << 3)
|
||||
#define ALPHA_IPI_AST (1UL << 4)
|
||||
#define ALPHA_IPI_PAUSE (1UL << 5)
|
||||
#define ALPHA_IPI_XCALL (1UL << 6)
|
||||
#define ALPHA_IPI_GENERIC (1UL << 7)
|
||||
#define ALPHA_IPI_AST (1UL << 3)
|
||||
#define ALPHA_IPI_PAUSE (1UL << 4)
|
||||
#define ALPHA_IPI_XCALL (1UL << 5)
|
||||
#define ALPHA_IPI_GENERIC (1UL << 6)
|
||||
|
||||
#define ALPHA_NIPIS 8 /* must not exceed 64 */
|
||||
#define ALPHA_NIPIS 7 /* must not exceed 64 */
|
||||
|
||||
struct cpu_info;
|
||||
struct trapframe;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* $NetBSD: pmap.h,v 1.82 2020/07/23 19:23:27 skrll Exp $ */
|
||||
/* $NetBSD: pmap.h,v 1.83 2020/08/29 20:07:00 thorpej Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1998, 1999, 2000, 2001, 2007 The NetBSD Foundation, Inc.
|
||||
@ -123,35 +123,41 @@
|
||||
*
|
||||
* Note pm_asn and pm_asngen are arrays allocated in pmap_create().
|
||||
* Their size is based on the PCS count from the HWRPB, and indexed
|
||||
* by processor ID (from `whami').
|
||||
* by processor ID (from `whami'). This is all padded to COHERENCY_UNIT
|
||||
* to avoid false sharing.
|
||||
*
|
||||
* The kernel pmap is a special case; it gets statically-allocated
|
||||
* The kernel pmap is a special case; since the kernel uses only ASM
|
||||
* mappings and uses a reserved ASN to keep the TLB clean, we don't
|
||||
* allocate any ASN info for the kernel pmap at all.
|
||||
* arrays which hold enough for ALPHA_MAXPROCS.
|
||||
*/
|
||||
struct pmap_asn_info {
|
||||
unsigned int pma_asn; /* address space number */
|
||||
unsigned int pma_pad0;
|
||||
unsigned long pma_asngen; /* ASN generation number */
|
||||
unsigned long pma_padN[(COHERENCY_UNIT / 8) - 2];
|
||||
};
|
||||
|
||||
struct pmap {
|
||||
TAILQ_ENTRY(pmap) pm_list; /* list of all pmaps */
|
||||
pt_entry_t *pm_lev1map; /* level 1 map */
|
||||
int pm_count; /* pmap reference count */
|
||||
kmutex_t pm_lock; /* lock on pmap */
|
||||
struct pmap_statistics pm_stats; /* pmap statistics */
|
||||
unsigned long pm_cpus; /* mask of CPUs using pmap */
|
||||
unsigned long pm_needisync; /* mask of CPUs needing isync */
|
||||
struct pmap_asn_info pm_asni[]; /* ASN information */
|
||||
struct pmap { /* pmaps are aligned to COHERENCY_UNIT boundaries */
|
||||
/* pmaps are locked by hashed mutexes */
|
||||
pt_entry_t *pm_lev1map; /* [ 0] level 1 map */
|
||||
unsigned long pm_cpus; /* [ 8] CPUs using pmap */
|
||||
unsigned long pm_needisync; /* [16] CPUs needing isync */
|
||||
struct pmap_statistics pm_stats; /* [32] statistics */
|
||||
long pm_count; /* [40] reference count */
|
||||
TAILQ_ENTRY(pmap) pm_list; /* [48] list of all pmaps */
|
||||
/* -- COHERENCY_UNIT boundary -- */
|
||||
struct pmap_asn_info pm_asni[]; /* [64] ASN information */
|
||||
/* variable length */
|
||||
};
|
||||
|
||||
/*
|
||||
* Compute the sizeof of a pmap structure.
|
||||
*/
|
||||
#define PMAP_SIZEOF(x) \
|
||||
(ALIGN(offsetof(struct pmap, pm_asni[(x)])))
|
||||
|
||||
#define PMAP_ASN_RESERVED 0 /* reserved for Lev1map users */
|
||||
#define PMAP_ASN_KERNEL 0 /* kernel-reserved ASN */
|
||||
#define PMAP_ASN_FIRST_USER 1 /* first user ASN */
|
||||
#define PMAP_ASNGEN_INVALID 0 /* reserved (invalid) ASN generation */
|
||||
#define PMAP_ASNGEN_INITIAL 1 /* first valid generatation */
|
||||
|
||||
/*
|
||||
* For each struct vm_page, there is a list of all currently valid virtual
|
||||
@ -188,22 +194,12 @@ typedef struct pv_entry {
|
||||
#define _PMAP_MAY_USE_PROM_CONSOLE
|
||||
#endif
|
||||
|
||||
#if defined(MULTIPROCESSOR)
|
||||
struct cpu_info;
|
||||
struct trapframe;
|
||||
|
||||
void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, u_long *);
|
||||
void pmap_tlb_shootnow(u_long);
|
||||
void pmap_do_tlb_shootdown(struct cpu_info *, struct trapframe *);
|
||||
#define PMAP_TLB_SHOOTDOWN_CPUSET_DECL u_long shootset = 0;
|
||||
#define PMAP_TLB_SHOOTDOWN(pm, va, pte) \
|
||||
pmap_tlb_shootdown((pm), (va), (pte), &shootset)
|
||||
#define PMAP_TLB_SHOOTNOW() \
|
||||
pmap_tlb_shootnow(shootset)
|
||||
#else
|
||||
#define PMAP_TLB_SHOOTDOWN_CPUSET_DECL /* nothing */
|
||||
#define PMAP_TLB_SHOOTDOWN(pm, va, pte) /* nothing */
|
||||
#define PMAP_TLB_SHOOTNOW() /* nothing */
|
||||
void pmap_init_cpu(struct cpu_info *);
|
||||
#if defined(MULTIPROCESSOR)
|
||||
void pmap_tlb_shootdown_ipi(struct cpu_info *, struct trapframe *);
|
||||
#endif /* MULTIPROCESSOR */
|
||||
|
||||
#define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count)
|
||||
@ -330,17 +326,6 @@ pmap_l3pte(pmap_t pmap, vaddr_t v, pt_entry_t *l2pte)
|
||||
return (&lev3map[l3pte_index(v)]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Macros for locking pmap structures.
|
||||
*
|
||||
* Note that we if we access the kernel pmap in interrupt context, it
|
||||
* is only to update statistics. Since stats are updated using atomic
|
||||
* operations, locking the kernel pmap is not necessary. Therefore,
|
||||
* it is not necessary to block interrupts when locking pmap strucutres.
|
||||
*/
|
||||
#define PMAP_LOCK(pmap) mutex_enter(&(pmap)->pm_lock)
|
||||
#define PMAP_UNLOCK(pmap) mutex_exit(&(pmap)->pm_lock)
|
||||
|
||||
/*
|
||||
* Macro for processing deferred I-stream synchronization.
|
||||
*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user