NetBSD/sys/kern/subr_cpu.c

/*	$NetBSD: subr_cpu.c,v 1.22 2024/03/05 20:59:41 thorpej Exp $	*/

/*-
 * Copyright (c) 2007, 2008, 2009, 2010, 2012, 2019, 2020
 *     The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*-
 * Copyright (c)2007 YAMAMOTO Takashi,
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * CPU related routines shared with rump.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_cpu.c,v 1.22 2024/03/05 20:59:41 thorpej Exp $");

#include <sys/param.h>
#include <sys/atomic.h>
#include <sys/systm.h>
#include <sys/sched.h>
#include <sys/conf.h>
#include <sys/cpu.h>
#include <sys/proc.h>
#include <sys/kernel.h>
#include <sys/kmem.h>

static void	cpu_topology_fake1(struct cpu_info *);

kmutex_t	cpu_lock		__cacheline_aligned;
int		ncpu			__read_mostly;
int		ncpuonline		__read_mostly;
bool		mp_online		__read_mostly;
static bool	cpu_topology_present	__read_mostly;
static bool	cpu_topology_haveslow	__read_mostly;
int64_t		cpu_counts[CPU_COUNT_MAX];

/* An array of CPUs.  There are ncpu entries. */
struct cpu_info **cpu_infos		__read_mostly;

/* Note: set on mi_cpu_attach() and idle_loop(). */
kcpuset_t *	kcpuset_attached	__read_mostly	= NULL;
kcpuset_t *	kcpuset_running		__read_mostly	= NULL;

static char cpu_model[128];

/*
 * mi_cpu_init: early initialisation of MI CPU related structures.
 *
 * Note: may not block and memory allocator is not yet available.
 */
void
mi_cpu_init(void)
{
	struct cpu_info *ci;

	mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE);

	kcpuset_create(&kcpuset_attached, true);
	kcpuset_create(&kcpuset_running, true);
	kcpuset_set(kcpuset_running, 0);

	ci = curcpu();
	cpu_topology_fake1(ci);
}

int
cpu_setmodel(const char *fmt, ...)
{
	int len;
	va_list ap;

	va_start(ap, fmt);
	len = vsnprintf(cpu_model, sizeof(cpu_model), fmt, ap);
	va_end(ap);
	return len;
}

const char *
cpu_getmodel(void)
{
	return cpu_model;
}

bool
cpu_softintr_p(void)
{

	return (curlwp->l_pflag & LP_INTR) != 0;
}

bool
curcpu_stable(void)
{
	struct lwp *const l = curlwp;
	const int pflag = l->l_pflag;
	const int nopreempt = l->l_nopreempt;

	/*
	 * - Softints (LP_INTR) never migrate between CPUs.
	 * - Bound lwps (LP_BOUND), either kthreads created bound to
	 *   a CPU or any lwps bound with curlwp_bind, never migrate.
	 * - If kpreemption is disabled, the lwp can't migrate.
	 * - If we're in interrupt context, preemption is blocked.
	 *
	 * We combine the LP_INTR, LP_BOUND, and l_nopreempt test into
	 * a single predicted-true branch so this is cheap to assert in
	 * most contexts where it will be used, then fall back to
	 * calling the full kpreempt_disabled() and cpu_intr_p() as
	 * subroutines.
	 *
	 * XXX Is cpu_intr_p redundant with kpreempt_disabled?
	 */
	return __predict_true(((pflag & (LP_INTR|LP_BOUND)) | nopreempt)
		!= 0) ||
	    kpreempt_disabled() ||
	    cpu_intr_p();
}

/*
 * Collect CPU topology information as each CPU is attached.  This can be
 * called early during boot, so we need to be careful what we do.
 */
void
cpu_topology_set(struct cpu_info *ci, u_int package_id, u_int core_id,
    u_int smt_id, u_int numa_id)
{
	enum cpu_rel rel;

	cpu_topology_present = true;
	ci->ci_package_id = package_id;
	ci->ci_core_id = core_id;
	ci->ci_smt_id = smt_id;
	ci->ci_numa_id = numa_id;
	for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
		ci->ci_sibling[rel] = ci;
		ci->ci_nsibling[rel] = 1;
	}
}

/*
 * Collect CPU relative speed
 */
void
cpu_topology_setspeed(struct cpu_info *ci, bool slow)
{

	cpu_topology_haveslow |= slow;
	ci->ci_is_slow = slow;
}

/*
 * Link a CPU into the given circular list.
 */
static void
cpu_topology_link(struct cpu_info *ci, struct cpu_info *ci2, enum cpu_rel rel)
{
	struct cpu_info *ci3;

	/* Walk to the end of the existing circular list and append. */
	for (ci3 = ci2;; ci3 = ci3->ci_sibling[rel]) {
		ci3->ci_nsibling[rel]++;
		if (ci3->ci_sibling[rel] == ci2) {
			break;
		}
	}
	ci->ci_sibling[rel] = ci2;
	ci3->ci_sibling[rel] = ci;
	ci->ci_nsibling[rel] = ci3->ci_nsibling[rel];
}

/*
 * Print out the topology lists.
 */
static void
cpu_topology_dump(void)
{
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci, *ci2;
	const char *names[] = { "core", "pkg", "1st" };
	enum cpu_rel rel;
	int i;

	CTASSERT(__arraycount(names) >= __arraycount(ci->ci_sibling));
	if (ncpu == 1) {
		return;
	}

	for (CPU_INFO_FOREACH(cii, ci)) {
		if (cpu_topology_haveslow)
			aprint_debug("%s ", ci->ci_is_slow ? "slow" : "fast");
		for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
			aprint_debug("%s has %d %s siblings:", cpu_name(ci),
			    ci->ci_nsibling[rel], names[rel]);
			ci2 = ci->ci_sibling[rel];
			i = 0;
			do {
				aprint_debug(" %s", cpu_name(ci2));
				ci2 = ci2->ci_sibling[rel];
			} while (++i < 64 && ci2 != ci->ci_sibling[rel]);
			if (i == 64) {
				aprint_debug(" GAVE UP");
			}
			aprint_debug("\n");
		}
		aprint_debug("%s first in package: %s\n", cpu_name(ci),
		    cpu_name(ci->ci_package1st));
	}
}

/*
 * Fake up topology info if we have none, or if what we got was bogus.
 * Used early in boot, and by cpu_topology_fake().
 */
static void
cpu_topology_fake1(struct cpu_info *ci)
{
	enum cpu_rel rel;

	for (rel = 0; rel < __arraycount(ci->ci_sibling); rel++) {
		ci->ci_sibling[rel] = ci;
		ci->ci_nsibling[rel] = 1;
	}
	if (!cpu_topology_present) {
		ci->ci_package_id = cpu_index(ci);
	}
	ci->ci_schedstate.spc_flags |=
	    (SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS);
	ci->ci_package1st = ci;
	if (!cpu_topology_haveslow) {
		ci->ci_is_slow = false;
	}
}

/*
 * Fake up topology info if we have none, or if what we got was bogus.
 * Don't override ci_package_id, etc, if cpu_topology_present is set.
 * MD code also uses these.
 */
static void
cpu_topology_fake(void)
{
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;

	for (CPU_INFO_FOREACH(cii, ci)) {
		cpu_topology_fake1(ci);
		/* Undo (early boot) flag set so everything links OK. */
		ci->ci_schedstate.spc_flags &=
		    ~(SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS);
	}
}

/*
 * Fix up basic CPU topology info.  Right now that means attach each CPU to
 * circular lists of its siblings in the same core, and in the same package.
 */
void
cpu_topology_init(void)
{
	CPU_INFO_ITERATOR cii, cii2;
	struct cpu_info *ci, *ci2, *ci3;
	u_int minsmt, mincore;

	if (!cpu_topology_present) {
		cpu_topology_fake();
		goto linkit;
	}

	/* Find siblings in same core and package. */
	for (CPU_INFO_FOREACH(cii, ci)) {
		ci->ci_schedstate.spc_flags &=
		    ~(SPCF_CORE1ST | SPCF_PACKAGE1ST | SPCF_1STCLASS);
		for (CPU_INFO_FOREACH(cii2, ci2)) {
			/* Avoid bad things happening. */
			if (ci2->ci_package_id == ci->ci_package_id &&
			    ci2->ci_core_id == ci->ci_core_id &&
			    ci2->ci_smt_id == ci->ci_smt_id &&
			    ci2 != ci) {
#ifdef DEBUG
				printf("cpu%u %p pkg %u core %u smt %u same as "
				       "cpu%u %p pkg %u core %u smt %u\n",
				       cpu_index(ci), ci, ci->ci_package_id,
				       ci->ci_core_id, ci->ci_smt_id,
				       cpu_index(ci2), ci2, ci2->ci_package_id,
				       ci2->ci_core_id, ci2->ci_smt_id);
#endif
			    	printf("cpu_topology_init: info bogus, "
			    	    "faking it\n");
			    	cpu_topology_fake();
			    	goto linkit;
			}
			if (ci2 == ci ||
			    ci2->ci_package_id != ci->ci_package_id) {
				continue;
			}
			/* Find CPUs in the same core. */
			if (ci->ci_nsibling[CPUREL_CORE] == 1 &&
			    ci->ci_core_id == ci2->ci_core_id) {
			    	cpu_topology_link(ci, ci2, CPUREL_CORE);
			}
			/* Find CPUs in the same package. */
			if (ci->ci_nsibling[CPUREL_PACKAGE] == 1) {
			    	cpu_topology_link(ci, ci2, CPUREL_PACKAGE);
			}
			if (ci->ci_nsibling[CPUREL_CORE] > 1 &&
			    ci->ci_nsibling[CPUREL_PACKAGE] > 1) {
				break;
			}
		}
	}

 linkit:
	/* Identify lowest numbered SMT in each core. */
	for (CPU_INFO_FOREACH(cii, ci)) {
		ci2 = ci3 = ci;
		minsmt = ci->ci_smt_id;
		do {
			if (ci2->ci_smt_id < minsmt) {
				ci3 = ci2;
				minsmt = ci2->ci_smt_id;
			}
			ci2 = ci2->ci_sibling[CPUREL_CORE];
		} while (ci2 != ci);
		ci3->ci_schedstate.spc_flags |= SPCF_CORE1ST;
	}

	/* Identify lowest numbered SMT in each package. */
	ci3 = NULL;
	for (CPU_INFO_FOREACH(cii, ci)) {
		if ((ci->ci_schedstate.spc_flags & SPCF_CORE1ST) == 0) {
			continue;
		}
		ci2 = ci3 = ci;
		mincore = ci->ci_core_id;
		do {
			if ((ci2->ci_schedstate.spc_flags &
			    SPCF_CORE1ST) != 0 &&
			    ci2->ci_core_id < mincore) {
				ci3 = ci2;
				mincore = ci2->ci_core_id;
			}
			ci2 = ci2->ci_sibling[CPUREL_PACKAGE];
		} while (ci2 != ci);

		if ((ci3->ci_schedstate.spc_flags & SPCF_PACKAGE1ST) != 0) {
			/* Already identified - nothing more to do. */
			continue;
		}
		ci3->ci_schedstate.spc_flags |= SPCF_PACKAGE1ST;

		/* Walk through all CPUs in package and point to first. */
		ci2 = ci3;
		do {
			ci2->ci_package1st = ci3;
			ci2->ci_sibling[CPUREL_PACKAGE1ST] = ci3;
			ci2 = ci2->ci_sibling[CPUREL_PACKAGE];
		} while (ci2 != ci3);

		/* Now look for somebody else to link to. */
		for (CPU_INFO_FOREACH(cii2, ci2)) {
			if ((ci2->ci_schedstate.spc_flags & SPCF_PACKAGE1ST)
			    != 0 && ci2 != ci3) {
			    	cpu_topology_link(ci3, ci2, CPUREL_PACKAGE1ST);
			    	break;
			}
		}
	}

	/* Walk through all packages, starting with value of ci3 from above. */
	KASSERT(ci3 != NULL);
	ci = ci3;
	do {
		/* Walk through CPUs in the package and copy in PACKAGE1ST. */
		ci2 = ci;
		do {
			ci2->ci_sibling[CPUREL_PACKAGE1ST] =
			    ci->ci_sibling[CPUREL_PACKAGE1ST];
			ci2->ci_nsibling[CPUREL_PACKAGE1ST] =
			    ci->ci_nsibling[CPUREL_PACKAGE1ST];
			ci2 = ci2->ci_sibling[CPUREL_PACKAGE];
		} while (ci2 != ci);
		ci = ci->ci_sibling[CPUREL_PACKAGE1ST];
	} while (ci != ci3);

	if (cpu_topology_haveslow) {
		/*
		 * For asymmetric systems where some CPUs are slower than
		 * others, mark first class CPUs for the scheduler.  This
		 * conflicts with SMT right now so whinge if observed.
		 */
		if (curcpu()->ci_nsibling[CPUREL_CORE] > 1) {
			printf("cpu_topology_init: asymmetric & SMT??\n");
		}
		for (CPU_INFO_FOREACH(cii, ci)) {
			if (!ci->ci_is_slow) {
				ci->ci_schedstate.spc_flags |= SPCF_1STCLASS;
			}
		}
	} else {
		/*
		 * For any other configuration mark the 1st CPU in each
		 * core as a first class CPU.
		 */
		for (CPU_INFO_FOREACH(cii, ci)) {
			if ((ci->ci_schedstate.spc_flags & SPCF_CORE1ST) != 0) {
				ci->ci_schedstate.spc_flags |= SPCF_1STCLASS;
			}
		}
	}

	cpu_topology_dump();
}

/*
 * Adjust one count, for a counter that's NOT updated from interrupt
 * context.  Hardly worth making an inline due to preemption stuff.
 */
void
cpu_count(enum cpu_count idx, int64_t delta)
{
	lwp_t *l = curlwp;
	KPREEMPT_DISABLE(l);
	l->l_cpu->ci_counts[idx] += delta;
	KPREEMPT_ENABLE(l);
}

/*
 * Fetch fresh sum total for all counts.  Expensive - don't call often.
 *
 * If poll is true, the caller is okay with less recent values (but
 * no more than 1/hz seconds old).  Where this is called very often that
 * should be the case.
 *
 * This should be reasonably quick so that any value collected get isn't
 * totally out of whack, and it can also be called from interrupt context,
 * so go to splvm() while summing the counters.  It's tempting to use a spin
 * mutex here but this routine is called from DDB.
 */
void
cpu_count_sync(bool poll)
{
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;
	int64_t sum[CPU_COUNT_MAX], *ptr;
	static int lasttick;
	int curtick, s;
	enum cpu_count i;

	KASSERT(sizeof(ci->ci_counts) == sizeof(cpu_counts));

	if (__predict_false(!mp_online)) {
		memcpy(cpu_counts, curcpu()->ci_counts, sizeof(cpu_counts));
		return;
	}

	s = splvm();
	curtick = getticks();
	if (poll && atomic_load_acquire(&lasttick) == curtick) {
		splx(s);
		return;
	}
	memset(sum, 0, sizeof(sum));
	curcpu()->ci_counts[CPU_COUNT_SYNC]++;
	for (CPU_INFO_FOREACH(cii, ci)) {
		ptr = ci->ci_counts;
		for (i = 0; i < CPU_COUNT_MAX; i += 8) {
			sum[i+0] += ptr[i+0];
			sum[i+1] += ptr[i+1];
			sum[i+2] += ptr[i+2];
			sum[i+3] += ptr[i+3];
			sum[i+4] += ptr[i+4];
			sum[i+5] += ptr[i+5];
			sum[i+6] += ptr[i+6];
			sum[i+7] += ptr[i+7];
		}
		KASSERT(i == CPU_COUNT_MAX);
	}
	memcpy(cpu_counts, sum, sizeof(cpu_counts));
	atomic_store_release(&lasttick, curtick);
	splx(s);
}