Pull up following revision(s) (requested by riastradh in ticket #1676):
sys/arch/ia64/ia64/vm_machdep.c: revision 1.18 sys/arch/powerpc/powerpc/locore_subr.S: revision 1.67 sys/arch/aarch64/aarch64/locore.S: revision 1.91 sys/arch/mips/include/asm.h: revision 1.74 sys/arch/hppa/include/cpu.h: revision 1.13 sys/arch/arm/arm/armv6_start.S: revision 1.38 sys/arch/evbmips/ingenic/cpu_startup.S: revision 1.2 sys/arch/mips/mips/locore.S: revision 1.229 sys/arch/aarch64/aarch64/cpuswitch.S: revision 1.40 sys/arch/alpha/include/asm.h: revision 1.45 sys/arch/sparc64/sparc64/locore.s: revision 1.432 sys/arch/vax/vax/subr.S: revision 1.42 sys/arch/mips/mips/locore_mips3.S: revision 1.116 sys/arch/ia64/ia64/machdep.c: revision 1.44 sys/arch/arm/arm32/cpuswitch.S: revision 1.106 sys/arch/sparc/sparc/locore.s: revision 1.284 (all via patch) aarch64: Add missing barriers in cpu_switchto. Details in comments. Note: This is a conservative change that inserts a barrier where there was a comment saying none is needed, which is probably correct. The goal of this change is to systematically add barriers to be confident in correctness; subsequent changes may remove some bariers, as an optimization, with an explanation of why each barrier is not needed. PR kern/57240 alpha: Add missing barriers in cpu_switchto. Details in comments. arm32: Add missing barriers in cpu_switchto. Details in comments. hppa: Add missing barriers in cpu_switchto. Not sure hppa has ever had working MULTIPROCESSOR, so maybe no pullups needed? ia64: Add missing barriers in cpu_switchto. (ia64 has never really worked, so no pullups needed, right?) mips: Add missing barriers in cpu_switchto. Details in comments. powerpc: Add missing barriers in cpu_switchto. Details in comments. sparc: Add missing barriers in cpu_switchto. sparc64: Add missing barriers in cpu_switchto. Details in comments. vax: Note where cpu_switchto needs barriers. Not sure vax has ever had working MULTIPROCESSOR, though, and I'm not even sure how to spell store-before-load barriers on VAX, so no functional change for now.
This commit is contained in:
martin
parent
b6bddc5a84
commit
3d0e3553b5
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: cpuswitch.S,v 1.11.4.1 2019/12/24 17:30:28 martin Exp $ */
|
||||
/* $NetBSD: cpuswitch.S,v 1.11.4.2 2023/07/31 13:44:15 martin Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2014 The NetBSD Foundation, Inc.
|
||||
|
|
@ -37,7 +37,7 @@
|
|||
#include "opt_ddb.h"
|
||||
#include "opt_kasan.h"
|
||||
|
||||
RCSID("$NetBSD: cpuswitch.S,v 1.11.4.1 2019/12/24 17:30:28 martin Exp $")
|
||||
RCSID("$NetBSD: cpuswitch.S,v 1.11.4.2 2023/07/31 13:44:15 martin Exp $")
|
||||
|
||||
/*
|
||||
* At IPL_SCHED:
|
||||
|
|
@ -81,7 +81,29 @@ ENTRY_NP(cpu_switchto)
|
|||
msr cpacr_el1, x5 /* restore cpacr_el1 */
|
||||
|
||||
mrs x3, tpidr_el1
|
||||
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
dmb ishst /* store-before-store */
|
||||
str x1, [x3, #CI_CURLWP] /* switch curlwp to new lwp */
|
||||
dmb ish /* store-before-load */
|
||||
|
||||
ENABLE_INTERRUPT
|
||||
|
||||
/*
|
||||
|
|
@ -133,7 +155,9 @@ ENTRY_NP(cpu_switchto_softint)
|
|||
ldr x6, [x19, #L_PCB] /* x6 = lwp_getpcb(curlwp) */
|
||||
str x4, [x6, #PCB_TF]
|
||||
str x5, [x19, #L_MD_CPACR]
|
||||
dmb ishst /* for mutex_enter; see cpu_switchto */
|
||||
str x0, [x3, #CI_CURLWP] /* curcpu()->ci_curlwp = softlwp; */
|
||||
dmb ish /* for mutex_enter; see cpu_switchto */
|
||||
|
||||
#ifdef KASAN
|
||||
/* clear the new stack */
|
||||
|
|
@ -155,7 +179,9 @@ ENTRY_NP(cpu_switchto_softint)
|
|||
|
||||
mrs x3, tpidr_el1
|
||||
DISABLE_INTERRUPT
|
||||
dmb ishst /* for mutex_enter; see cpu_switchto */
|
||||
str x19, [x3, #CI_CURLWP] /* curcpu()->ci_curlwp := x19 */
|
||||
dmb ish /* for mutex_enter; see cpu_switchto */
|
||||
ldr x6, [x19, #L_PCB] /* x6 = lwp_getpcb(curlwp) */
|
||||
ldr x4, [x6, #PCB_TF] /* x4 := pinned_lwp->l_addr->pcb_tf */
|
||||
ldr x5, [x19, #L_MD_CPACR] /* x5 := pinned_lwp->l_md_cpacr */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore.S,v 1.39.2.5 2020/01/21 11:11:00 martin Exp $ */
|
||||
/* $NetBSD: locore.S,v 1.39.2.6 2023/07/31 13:44:15 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
|
||||
|
|
@ -38,7 +38,7 @@
|
|||
#include <aarch64/hypervisor.h>
|
||||
#include "assym.h"
|
||||
|
||||
RCSID("$NetBSD: locore.S,v 1.39.2.5 2020/01/21 11:11:00 martin Exp $")
|
||||
RCSID("$NetBSD: locore.S,v 1.39.2.6 2023/07/31 13:44:15 martin Exp $")
|
||||
|
||||
#ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED
|
||||
#define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRnE
|
||||
|
|
@ -486,6 +486,11 @@ mp_vstart:
|
|||
mrs x0, tpidr_el1 /* curcpu() */
|
||||
|
||||
ldr x1, [x0, #CI_IDLELWP] /* x1 = curcpu()->ci_data.cpu_idlelwp */
|
||||
/*
|
||||
* No membar needed because we're not switching from a
|
||||
* previous lwp, and the idle lwp we're switching to can't be
|
||||
* holding locks already; see cpu_switchto.
|
||||
*/
|
||||
str x1, [x0, #CI_CURLWP] /* curlwp is idlelwp */
|
||||
|
||||
/* get my stack from lwp */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: armv6_start.S,v 1.14 2019/06/12 06:53:21 skrll Exp $ */
|
||||
/* $NetBSD: armv6_start.S,v 1.14.2.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2012, 2017, 2018 The NetBSD Foundation, Inc.
|
||||
|
|
@ -884,6 +884,11 @@ armv7_mpcontinuation:
|
|||
#else
|
||||
#error either TPIDRPRW_IS_CURCPU or TPIDRPRW_IS_CURLWP must be defined
|
||||
#endif
|
||||
/*
|
||||
* No membar needed because we're not switching from a
|
||||
* previous lwp, and the idle lwp we're switching to can't be
|
||||
* holding locks already; see cpu_switchto.
|
||||
*/
|
||||
str r6, [r5, #CI_CURLWP] // and note we are running on it
|
||||
|
||||
mov r0, r5 // pass cpu_info
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: cpuswitch.S,v 1.93 2018/11/22 21:28:21 skrll Exp $ */
|
||||
/* $NetBSD: cpuswitch.S,v 1.93.4.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright 2003 Wasabi Systems, Inc.
|
||||
|
|
@ -87,7 +87,7 @@
|
|||
#include <arm/asm.h>
|
||||
#include <arm/locore.h>
|
||||
|
||||
RCSID("$NetBSD: cpuswitch.S,v 1.93 2018/11/22 21:28:21 skrll Exp $")
|
||||
RCSID("$NetBSD: cpuswitch.S,v 1.93.4.1 2023/07/31 13:44:16 martin Exp $")
|
||||
|
||||
/* LINTSTUB: include <sys/param.h> */
|
||||
|
||||
|
|
@ -205,8 +205,34 @@ ENTRY(cpu_switchto)
|
|||
mcr p15, 0, r6, c13, c0, 4 /* set current lwp */
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
|
||||
/* We have a new curlwp now so make a note of it */
|
||||
#ifdef _ARM_ARCH_7
|
||||
dmb /* store-before-store */
|
||||
#endif
|
||||
str r6, [r5, #(CI_CURLWP)]
|
||||
#ifdef _ARM_ARCH_7
|
||||
dmb /* store-before-load */
|
||||
#endif
|
||||
|
||||
/* Get the new pcb */
|
||||
ldr r7, [r6, #(L_PCB)]
|
||||
|
||||
|
|
@ -402,8 +428,14 @@ ENTRY_NP(softint_switch)
|
|||
*/
|
||||
#if defined(TPIDRPRW_IS_CURLWP)
|
||||
mcr p15, 0, r5, c13, c0, 4 /* save new lwp */
|
||||
#endif
|
||||
#ifdef _ARM_ARCH_7
|
||||
dmb /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
str r5, [r7, #(CI_CURLWP)] /* save new lwp */
|
||||
#ifdef _ARM_ARCH_7
|
||||
dmb /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Normally, we'd get {r8-r13} but since this is a softint lwp
|
||||
|
|
@ -430,8 +462,14 @@ ENTRY_NP(softint_switch)
|
|||
IRQdisable
|
||||
#if defined(TPIDRPRW_IS_CURLWP)
|
||||
mcr p15, 0, r4, c13, c0, 4 /* restore pinned lwp */
|
||||
#endif
|
||||
#ifdef _ARM_ARCH_7
|
||||
dmb /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
str r4, [r7, #(CI_CURLWP)] /* restore pinned lwp */
|
||||
#ifdef _ARM_ARCH_7
|
||||
dmb /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
ldr sp, [r2, #(PCB_KSP)] /* now running on the old stack. */
|
||||
|
||||
/* At this point we can allow IRQ's again. */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: cpu_startup.S,v 1.1 2016/01/29 01:54:14 macallan Exp $ */
|
||||
/* $NetBSD: cpu_startup.S,v 1.1.26.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2015 Michael Lorenz
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
#include <sys/endian.h>
|
||||
|
||||
#include <mips/asm.h>
|
||||
RCSID("$NetBSD: cpu_startup.S,v 1.1 2016/01/29 01:54:14 macallan Exp $");
|
||||
RCSID("$NetBSD: cpu_startup.S,v 1.1.26.1 2023/07/31 13:44:16 martin Exp $");
|
||||
|
||||
#ifdef MULTIPROCESSOR
|
||||
|
||||
|
|
@ -56,6 +56,11 @@ NESTED_NOPROFILE(ingenic_trampoline, 0, ra)
|
|||
nop
|
||||
beqz MIPS_CURLWP, 1b
|
||||
nop
|
||||
/*
|
||||
* No membar needed because we're not switching from a
|
||||
* previous lwp, and the idle lwp we're switching to can't be
|
||||
* holding locks already; see cpu_switchto.
|
||||
*/
|
||||
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(a0)
|
||||
|
||||
li v0, 0
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: cpu.h,v 1.5 2019/04/16 12:25:17 skrll Exp $ */
|
||||
/* $NetBSD: cpu.h,v 1.5.4.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/* $OpenBSD: cpu.h,v 1.55 2008/07/23 17:39:35 kettenis Exp $ */
|
||||
|
||||
|
|
@ -199,7 +199,26 @@ extern int cpu_revision;
|
|||
#define GET_CURLWP(r) mfctl CR_CURCPU, r ! ldw CI_CURLWP(r), r
|
||||
#define GET_CURLWP_SPACE(s, r) mfctl CR_CURCPU, r ! ldw CI_CURLWP(s, r), r
|
||||
|
||||
#define SET_CURLWP(r,t) mfctl CR_CURCPU, t ! stw r, CI_CURLWP(t)
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
#define SET_CURLWP(r,t) \
|
||||
sync ! mfctl CR_CURCPU, t ! stw r, CI_CURLWP(t) ! sync
|
||||
|
||||
#else /* MULTIPROCESSOR */
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: machdep.c,v 1.43 2019/04/19 16:28:32 scole Exp $ */
|
||||
/* $NetBSD: machdep.c,v 1.43.4.1 2023/07/31 13:44:15 martin Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2003,2004 Marcel Moolenaar
|
||||
|
|
@ -638,6 +638,10 @@ ia64_init(void)
|
|||
|
||||
/*
|
||||
* Initialise process context. XXX: This should really be in cpu_switchto
|
||||
*
|
||||
* No membar needed because we're not switching from a
|
||||
* previous lwp, and the idle lwp we're switching to can't be
|
||||
* holding locks already; see cpu_switchto.
|
||||
*/
|
||||
ci->ci_curlwp = &lwp0;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: vm_machdep.c,v 1.16 2019/01/18 18:03:06 scole Exp $ */
|
||||
/* $NetBSD: vm_machdep.c,v 1.16.4.1 2023/07/31 13:44:15 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2006 The NetBSD Foundation, Inc.
|
||||
|
|
@ -37,6 +37,7 @@
|
|||
#include <sys/proc.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/cpu.h>
|
||||
#include <sys/atomic.h>
|
||||
|
||||
#include <machine/frame.h>
|
||||
#include <machine/md_var.h>
|
||||
|
|
@ -77,9 +78,29 @@ cpu_switchto(lwp_t *oldlwp, lwp_t *newlwp, bool returning)
|
|||
register uint64_t reg9 __asm("r9");
|
||||
|
||||
KASSERT(newlwp != NULL);
|
||||
|
||||
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
membar_producer(); /* store-before-store */
|
||||
ci->ci_curlwp = newlwp;
|
||||
|
||||
membar_sync(); /* store-before-load */
|
||||
|
||||
/* required for lwp_startup, copy oldlwp into r9, "mov r9=in0" */
|
||||
__asm __volatile("mov %0=%1" : "=r"(reg9) : "r"(oldlwp));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: asm.h,v 1.55 2018/09/04 00:01:41 mrg Exp $ */
|
||||
/* $NetBSD: asm.h,v 1.55.4.1 2023/07/31 13:44:15 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1992, 1993
|
||||
|
|
@ -511,6 +511,32 @@ _C_LABEL(x):
|
|||
#define NOP_L /* nothing */
|
||||
#endif
|
||||
|
||||
/* XXX pullup more mips barrier improvements here */
|
||||
#define SYNC_ACQ sync
|
||||
#define SYNC_REL sync
|
||||
|
||||
/*
|
||||
* Store-before-load barrier. Do not use this unless you know what
|
||||
* you're doing.
|
||||
*/
|
||||
#ifdef MULTIPROCESSOR
|
||||
#define SYNC_DEKKER sync
|
||||
#else
|
||||
#define SYNC_DEKKER /* nothing */
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Store-before-store and load-before-load barriers. These could be
|
||||
* made weaker than release (load/store-before-store) and acquire
|
||||
* (load-before-load/store) barriers, and newer MIPS does have
|
||||
* instruction encodings for finer-grained barriers like this, but I
|
||||
* dunno how to appropriately conditionalize their use or get the
|
||||
* assembler to be happy with them, so we'll use these definitions for
|
||||
* now.
|
||||
*/
|
||||
#define SYNC_PRODUCER SYNC_REL
|
||||
#define SYNC_CONSUMER SYNC_ACQ
|
||||
|
||||
/* CPU dependent hook for cp0 load delays */
|
||||
#if defined(MIPS1) || defined(MIPS2) || defined(MIPS3)
|
||||
#define MFC0_HAZARD sll $0,$0,1 /* super scalar nop */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore.S,v 1.219.4.1 2019/09/13 06:56:23 martin Exp $ */
|
||||
/* $NetBSD: locore.S,v 1.219.4.2 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1992, 1993
|
||||
|
|
@ -63,7 +63,7 @@
|
|||
#include <mips/trap.h>
|
||||
#include <mips/locore.h>
|
||||
|
||||
RCSID("$NetBSD: locore.S,v 1.219.4.1 2019/09/13 06:56:23 martin Exp $")
|
||||
RCSID("$NetBSD: locore.S,v 1.219.4.2 2023/07/31 13:44:16 martin Exp $")
|
||||
|
||||
#include "assym.h"
|
||||
|
||||
|
|
@ -290,7 +290,28 @@ NESTED(cpu_switchto, CALLFRAME_SIZ, ra)
|
|||
|
||||
PTR_L t2, L_CPU(MIPS_CURLWP)
|
||||
nop # patchable load delay slot
|
||||
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
SYNC_PRODUCER /* XXX fixup to nop for uniprocessor boot */
|
||||
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(t2)
|
||||
SYNC_DEKKER /* XXX fixup to nop for uniprocessor boot */
|
||||
|
||||
/* Check for restartable atomic sequences (RAS) */
|
||||
PTR_L a0, L_PROC(MIPS_CURLWP) # argument to ras_lookup
|
||||
|
|
@ -441,7 +462,9 @@ NESTED(softint_fast_dispatch, CALLFRAME_SIZ, ra)
|
|||
move MIPS_CURLWP, a0 # switch to softint lwp
|
||||
PTR_L s1, L_CPU(MIPS_CURLWP) # get curcpu()
|
||||
nop # patchable load delay slot
|
||||
SYNC_PRODUCER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
|
||||
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(s1) # ...
|
||||
SYNC_DEKKER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
|
||||
move s2, sp # remember sp
|
||||
move s3, t0 # remember curpcb
|
||||
|
||||
|
|
@ -452,7 +475,9 @@ NESTED(softint_fast_dispatch, CALLFRAME_SIZ, ra)
|
|||
|
||||
move sp, s2 # restore stack
|
||||
move MIPS_CURLWP, s0 # restore curlwp
|
||||
SYNC_PRODUCER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
|
||||
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(s1) # ....
|
||||
SYNC_DEKKER /* XXX fixup */ /* for mutex_enter; see cpu_switchto */
|
||||
|
||||
REG_L ra, CALLFRAME_RA(sp) # load early since we use it
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore_mips3.S,v 1.114 2018/01/26 05:29:43 maya Exp $ */
|
||||
/* $NetBSD: locore_mips3.S,v 1.114.8.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1997 Jonathan Stone (hereinafter referred to as the author)
|
||||
|
|
@ -92,7 +92,7 @@
|
|||
#include <mips/asm.h>
|
||||
#include <mips/cpuregs.h>
|
||||
|
||||
RCSID("$NetBSD: locore_mips3.S,v 1.114 2018/01/26 05:29:43 maya Exp $")
|
||||
RCSID("$NetBSD: locore_mips3.S,v 1.114.8.1 2023/07/31 13:44:16 martin Exp $")
|
||||
|
||||
#include "assym.h"
|
||||
|
||||
|
|
@ -793,6 +793,11 @@ NESTED_NOPROFILE(cpu_trampoline, 0, ra)
|
|||
nop
|
||||
beqz MIPS_CURLWP, 1b
|
||||
nop
|
||||
/*
|
||||
* No membar needed because we're not switching from a
|
||||
* previous lwp, and the idle lwp we're switching to can't be
|
||||
* holding locks already; see cpu_switchto.
|
||||
*/
|
||||
PTR_S MIPS_CURLWP, CPU_INFO_CURLWP(a0)
|
||||
|
||||
#ifdef _LP64
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore_subr.S,v 1.57.4.2 2020/03/03 18:54:59 martin Exp $ */
|
||||
/* $NetBSD: locore_subr.S,v 1.57.4.3 2023/07/31 13:44:15 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2001 Wasabi Systems, Inc.
|
||||
|
|
@ -224,7 +224,32 @@ switchto_restore:
|
|||
*/
|
||||
|
||||
GET_CPUINFO(%r7)
|
||||
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
#ifdef MULTIPROCESSOR
|
||||
sync /* store-before-store XXX use eieio if available -- cheaper */
|
||||
#endif
|
||||
stptr %r31,CI_CURLWP(%r7)
|
||||
#ifdef MULTIPROCESSOR
|
||||
sync /* store-before-load */
|
||||
#endif
|
||||
mr %r13,%r31
|
||||
#ifdef PPC_BOOKE
|
||||
mtsprg2 %r31 /* save curlwp in sprg2 */
|
||||
|
|
@ -400,7 +425,13 @@ _ENTRY(softint_fast_dispatch)
|
|||
* to a kernel thread
|
||||
*/
|
||||
|
||||
#ifdef MULTIPROCESSOR
|
||||
sync /* XXX eieio */ /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
stptr %r3, CI_CURLWP(%r7)
|
||||
#ifdef MULTIPROCESSOR
|
||||
sync /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
mr %r13, %r3
|
||||
#ifdef PPC_BOOKE
|
||||
mtsprg2 %r3
|
||||
|
|
@ -434,7 +465,13 @@ _ENTRY(softint_fast_dispatch)
|
|||
#endif
|
||||
|
||||
GET_CPUINFO(%r7)
|
||||
#ifdef MULTIPROCESSOR
|
||||
sync /* XXX eieio */ /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
stptr %r30, CI_CURLWP(%r7)
|
||||
#ifdef MULTIPROCESSOR
|
||||
sync /* for mutex_enter; see cpu_switchto */
|
||||
#endif
|
||||
mr %r13, %r30
|
||||
#ifdef PPC_BOOKE
|
||||
mtsprg2 %r30
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore.s,v 1.274 2019/06/07 00:18:26 mrg Exp $ */
|
||||
/* $NetBSD: locore.s,v 1.274.2.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1996 Paul Kranenburg
|
||||
|
|
@ -4904,7 +4904,30 @@ Lnosaveoldlwp:
|
|||
/* set new cpcb, and curlwp */
|
||||
sethi %hi(curlwp), %l7
|
||||
st %g5, [%l6 + %lo(cpcb)] ! cpcb = newpcb;
|
||||
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
/* stbar -- store-before-store, not needed on TSO */
|
||||
st %g3, [%l7 + %lo(curlwp)] ! curlwp = l;
|
||||
#ifdef MULTIPROCESSOR
|
||||
ldstub [%sp - 4], %g0 /* makeshift store-before-load barrier */
|
||||
#endif
|
||||
|
||||
/* compute new wim */
|
||||
ld [%g5 + PCB_WIM], %o0
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: locore.s,v 1.421 2019/07/18 18:21:45 palle Exp $ */
|
||||
/* $NetBSD: locore.s,v 1.421.2.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2006-2010 Matthew R. Green
|
||||
|
|
@ -6558,9 +6558,28 @@ ENTRY(cpu_switchto)
|
|||
* Load the new lwp. To load, we must change stacks and
|
||||
* alter cpcb and the window control registers, hence we must
|
||||
* keep interrupts disabled.
|
||||
*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*/
|
||||
|
||||
membar #StoreStore
|
||||
STPTR %i1, [%l7 + %lo(CURLWP)] ! curlwp = l;
|
||||
membar #StoreLoad
|
||||
STPTR %l1, [%l6 + %lo(CPCB)] ! cpcb = newpcb;
|
||||
|
||||
ldx [%l1 + PCB_SP], %i6
|
||||
|
|
@ -6653,7 +6672,9 @@ ENTRY(softint_fastintr)
|
|||
sethi %hi(USPACE - TF_SIZE - CC64FSZ - STKB), %o3
|
||||
LDPTR [%i0 + L_PCB], %l1 ! l1 = softint pcb
|
||||
or %o3, %lo(USPACE - TF_SIZE - CC64FSZ - STKB), %o3
|
||||
membar #StoreStore /* for mutex_enter; see cpu_switchto */
|
||||
STPTR %i0, [%l7 + %lo(CURLWP)]
|
||||
membar #StoreLoad /* for mutex_enter; see cpu_switchto */
|
||||
add %l1, %o3, %i6
|
||||
STPTR %l1, [%l6 + %lo(CPCB)]
|
||||
stx %i6, [%l1 + PCB_SP]
|
||||
|
|
@ -6666,7 +6687,9 @@ ENTRY(softint_fastintr)
|
|||
|
||||
/* switch back to interrupted lwp */
|
||||
ldx [%l5 + PCB_SP], %i6
|
||||
membar #StoreStore /* for mutex_enter; see cpu_switchto */
|
||||
STPTR %l0, [%l7 + %lo(CURLWP)]
|
||||
membar #StoreLoad /* for mutex_enter; see cpu_switchto */
|
||||
STPTR %l5, [%l6 + %lo(CPCB)]
|
||||
|
||||
restore ! rewind register window
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: subr.S,v 1.36 2019/04/06 03:06:28 thorpej Exp $ */
|
||||
/* $NetBSD: subr.S,v 1.36.4.1 2023/07/31 13:44:16 martin Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1994 Ludd, University of Lule}, Sweden.
|
||||
|
|
@ -316,7 +316,9 @@ softint_process:
|
|||
movab softint_exit,PCB_PC(%r3)/* do a quick exit */
|
||||
#ifdef MULTIPROCESSOR
|
||||
movl L_CPU(%r6),%r8
|
||||
/* XXX store-before-store barrier -- see cpu_switchto */
|
||||
movl %r6,CI_CURLWP(%r8)
|
||||
/* XXX store-before-load barrier -- see cpu_switchto */
|
||||
#endif
|
||||
|
||||
mtpr PCB_PADDR(%r3),$PR_PCBB /* restore PA of interrupted pcb */
|
||||
|
|
@ -339,7 +341,9 @@ softint_common:
|
|||
movl %r6,PCB_R6(%r3) /* move old lwp into new pcb */
|
||||
movl %r1,PCB_R7(%r3) /* move IPL into new pcb */
|
||||
#ifdef MULTIPROCESSOR
|
||||
/* XXX store-before-store barrier -- see cpu_switchto */
|
||||
movl %r2,CI_CURLWP(%r8) /* update ci_curlwp */
|
||||
/* XXX store-before-load barrier -- see cpu_switchto */
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
|
@ -405,7 +409,31 @@ JSBENTRY(Swtchto)
|
|||
#ifdef MULTIPROCESSOR
|
||||
movl L_CPU(%r0), %r8 /* get cpu_info of old lwp */
|
||||
movl %r8, L_CPU(%r1) /* update cpu_info of new lwp */
|
||||
/*
|
||||
* Issue barriers to coordinate mutex_exit on this CPU with
|
||||
* mutex_vector_enter on another CPU.
|
||||
*
|
||||
* 1. Any prior mutex_exit by oldlwp must be visible to other
|
||||
* CPUs before we set ci_curlwp := newlwp on this one,
|
||||
* requiring a store-before-store barrier.
|
||||
*
|
||||
* 2. ci_curlwp := newlwp must be visible on all other CPUs
|
||||
* before any subsequent mutex_exit by newlwp can even test
|
||||
* whether there might be waiters, requiring a
|
||||
* store-before-load barrier.
|
||||
*
|
||||
* See kern_mutex.c for details -- this is necessary for
|
||||
* adaptive mutexes to detect whether the lwp is on the CPU in
|
||||
* order to safely block without requiring atomic r/m/w in
|
||||
* mutex_exit.
|
||||
*
|
||||
* XXX I'm fuzzy on the memory model of VAX. I would guess
|
||||
* it's TSO like x86 but I can't find a store-before-load
|
||||
* barrier, which is the only one TSO requires explicitly.
|
||||
*/
|
||||
/* XXX store-before-store barrier */
|
||||
movl %r1,CI_CURLWP(%r8) /* update ci_curlwp */
|
||||
/* XXX store-before-load barrier */
|
||||
#endif
|
||||
|
||||
mtpr PCB_PADDR(%r3),$PR_PCBB # set PA of new pcb
|
||||
|
|
|
|||
Loading…
Reference in New Issue