diff --git a/sys/kern/kern_heartbeat.c b/sys/kern/kern_heartbeat.c index 852ef5d1d991..8d4ea01880d1 100644 --- a/sys/kern/kern_heartbeat.c +++ b/sys/kern/kern_heartbeat.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_heartbeat.c,v 1.9 2023/09/02 17:44:41 riastradh Exp $ */ +/* $NetBSD: kern_heartbeat.c,v 1.10 2023/09/06 12:29:14 riastradh Exp $ */ /*- * Copyright (c) 2023 The NetBSD Foundation, Inc. @@ -82,7 +82,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: kern_heartbeat.c,v 1.9 2023/09/02 17:44:41 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_heartbeat.c,v 1.10 2023/09/06 12:29:14 riastradh Exp $"); #ifdef _KERNEL_OPT #include "opt_ddb.h" @@ -132,21 +132,26 @@ void *heartbeat_sih __read_mostly; * * Called after the current CPU has been marked offline but before * it has stopped running, or after IPL has been raised for - * polling-mode console input. Caller must have preemption - * disabled. Non-nestable. Reversed by heartbeat_resume. + * polling-mode console input. Binds to the current CPU as a side + * effect. Nestable (but only up to 2^32 times, so don't do this + * in a loop). Reversed by heartbeat_resume. */ void heartbeat_suspend(void) { - struct cpu_info *ci = curcpu(); - int s; + unsigned *p; - KASSERT(curcpu_stable()); - KASSERT((ci->ci_schedstate.spc_flags & SPCF_HEARTBEATSUSPENDED) == 0); - - s = splsched(); - ci->ci_schedstate.spc_flags |= SPCF_HEARTBEATSUSPENDED; - splx(s); + /* + * We could use curlwp_bind, but we'd have to record whether we + * were already bound or not to pass to curlwp_bindx in + * heartbeat_resume. Using kpreempt_disable is simpler and + * unlikely to have any adverse consequences, since this only + * happens when we're about to go into a tight polling loop at + * raised IPL anyway. + */ + kpreempt_disable(); + p = &curcpu()->ci_heartbeat_suspend; + atomic_store_relaxed(p, *p + 1); } /* @@ -180,26 +185,26 @@ heartbeat_resume_cpu(struct cpu_info *ci) * * Called after the current CPU has started running but before it * has been marked online, or when ending polling-mode input - * before IPL is restored. Caller must have preemption disabled. + * before IPL is restored. Reverses heartbeat_suspend. */ void heartbeat_resume(void) { struct cpu_info *ci = curcpu(); + unsigned *p; int s; - KASSERT(curcpu_stable()); - KASSERT(ci->ci_schedstate.spc_flags & SPCF_HEARTBEATSUSPENDED); - /* - * Block heartbeats while we reset the state so we don't - * spuriously think we had a heart attack in the middle of - * resetting the count and the uptime stamp. + * Reset the state so nobody spuriously thinks we had a heart + * attack as soon as the heartbeat checks resume. */ s = splsched(); - ci->ci_schedstate.spc_flags &= ~SPCF_HEARTBEATSUSPENDED; heartbeat_resume_cpu(ci); splx(s); + + p = &ci->ci_heartbeat_suspend; + atomic_store_relaxed(p, *p - 1); + kpreempt_enable(); } /* @@ -223,10 +228,8 @@ heartbeat_timecounter_suspended(void) * iterating over all CPUs. */ for (CPU_INFO_FOREACH(cii, ci)) { - if (CPU_IS_PRIMARY(ci)) { - return ci->ci_schedstate.spc_flags & - SPCF_HEARTBEATSUSPENDED; - } + if (CPU_IS_PRIMARY(ci)) + return atomic_load_relaxed(&ci->ci_heartbeat_suspend); } /* @@ -551,7 +554,7 @@ select_patient(void) * in the iteration order. */ for (CPU_INFO_FOREACH(cii, ci)) { - if (ci->ci_schedstate.spc_flags & SPCF_HEARTBEATSUSPENDED) + if (atomic_load_relaxed(&ci->ci_heartbeat_suspend)) continue; if (passedcur) { /* @@ -628,8 +631,7 @@ heartbeat(void) period_secs = atomic_load_relaxed(&heartbeat_max_period_secs); if (__predict_false(period_ticks == 0) || __predict_false(period_secs == 0) || - __predict_false(curcpu()->ci_schedstate.spc_flags & - SPCF_HEARTBEATSUSPENDED)) + __predict_false(curcpu()->ci_heartbeat_suspend)) return; /* @@ -716,8 +718,7 @@ heartbeat(void) d = uptime - atomic_load_relaxed(&patient->ci_heartbeat_uptime_cache); if (__predict_false(d > period_secs) && __predict_false(d < UINT_MAX/2) && - ((patient->ci_schedstate.spc_flags & SPCF_HEARTBEATSUSPENDED) - == 0)) + atomic_load_relaxed(&patient->ci_heartbeat_suspend) == 0) defibrillate(patient, d); } @@ -737,16 +738,6 @@ db_read_unsigned(const volatile unsigned *p) return x; } -static int -db_read_signed(const volatile int *p) -{ - int x; - - db_read_bytes((db_addr_t)(uintptr_t)p, sizeof(x), (char *)&x); - - return x; -} - void heartbeat_dump(void) { @@ -754,13 +745,12 @@ heartbeat_dump(void) db_printf("Heartbeats:\n"); for (ci = db_cpu_first(); ci != NULL; ci = db_cpu_next(ci)) { - db_printf("cpu%u: count %u uptime %u stamp %u%s\n", + db_printf("cpu%u: count %u uptime %u stamp %u suspend %u\n", db_read_unsigned(&ci->ci_index), db_read_unsigned(&ci->ci_heartbeat_count), db_read_unsigned(&ci->ci_heartbeat_uptime_cache), db_read_unsigned(&ci->ci_heartbeat_uptime_stamp), - (db_read_signed(&ci->ci_schedstate.spc_flags) & - SPCF_HEARTBEATSUSPENDED ? " (suspended)" : "")); + db_read_unsigned(&ci->ci_heartbeat_suspend)); } } #endif diff --git a/sys/sys/cpu_data.h b/sys/sys/cpu_data.h index b6de36198741..3525f460e0c7 100644 --- a/sys/sys/cpu_data.h +++ b/sys/sys/cpu_data.h @@ -1,4 +1,4 @@ -/* $NetBSD: cpu_data.h,v 1.54 2023/07/13 12:06:20 riastradh Exp $ */ +/* $NetBSD: cpu_data.h,v 1.55 2023/09/06 12:29:14 riastradh Exp $ */ /*- * Copyright (c) 2004, 2006, 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc. @@ -194,6 +194,7 @@ struct cpu_data { unsigned cpu_heartbeat_uptime_cache; /* last time_uptime */ unsigned cpu_heartbeat_uptime_stamp; /* heartbeats since * uptime changed */ + unsigned cpu_heartbeat_suspend; /* suspend depth */ }; #define ci_schedstate ci_data.cpu_schedstate @@ -225,6 +226,7 @@ struct cpu_data { #define ci_heartbeat_count ci_data.cpu_heartbeat_count #define ci_heartbeat_uptime_cache ci_data.cpu_heartbeat_uptime_cache #define ci_heartbeat_uptime_stamp ci_data.cpu_heartbeat_uptime_stamp +#define ci_heartbeat_suspend ci_data.cpu_heartbeat_suspend #define cpu_nsyscall cpu_counts[CPU_COUNT_NSYSCALL] #define cpu_ntrap cpu_counts[CPU_COUNT_NTRAP] diff --git a/sys/sys/sched.h b/sys/sys/sched.h index 1fa14d942e72..8678aa21858b 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -1,4 +1,4 @@ -/* $NetBSD: sched.h,v 1.93 2023/09/02 17:43:37 riastradh Exp $ */ +/* $NetBSD: sched.h,v 1.94 2023/09/06 12:29:14 riastradh Exp $ */ /*- * Copyright (c) 1999, 2000, 2001, 2002, 2007, 2008, 2019, 2020 @@ -188,7 +188,6 @@ struct schedstate_percpu { #define SPCF_1STCLASS 0x0040 /* first class scheduling entity */ #define SPCF_CORE1ST 0x0100 /* first CPU in core */ #define SPCF_PACKAGE1ST 0x0200 /* first CPU in package */ -#define SPCF_HEARTBEATSUSPENDED 0x0400 /* heartbeat (temporarily) suspended */ #define SPCF_SWITCHCLEAR (SPCF_SEENRR|SPCF_SHOULDYIELD)