NetBSD/sys/kern/kern_softint.c

/*	$NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $	*/

/*-
 * Copyright (c) 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Generic software interrupt framework.
 *
 * Overview
 *
 *	The soft interrupt framework provides a mechanism to schedule a
 *	low priority callback that runs with thread context.  It allows
 *	for dynamic registration of software interrupts, and for fair
 *	queueing and prioritization of those interrupts.  The callbacks
 *	can be scheduled to run from nearly any point in the kernel: by
 *	code running with thread context, by code running from a
 *	hardware interrupt handler, and at any interrupt priority
 *	level.
 *
 * Priority levels
 *
 *	Since soft interrupt dispatch can be tied to the underlying
 *	architecture's interrupt dispatch code, it can be limited
 *	both by the capabilities of the hardware and the capabilities
 *	of the interrupt dispatch code itself.  The number of priority
 *	levels is restricted to four.  In order of priority (lowest to
 *	highest) the levels are: clock, bio, net, serial.
 *
 *	The names are symbolic and in isolation do not have any direct
 *	connection with a particular kind of device activity: they are
 *	only meant as a guide.
 *
 *	The four priority levels map directly to scheduler priority
 *	levels, and where the architecture implements 'fast' software
 *	interrupts, they also map onto interrupt priorities.  The
 *	interrupt priorities are intended to be hidden from machine
 *	independent code, which should use thread-safe mechanisms to
 *	synchronize with software interrupts (for example: mutexes).
 *
 * Capabilities
 *
 *	Software interrupts run with limited machine context.  In
 *	particular, they do not posess any address space context.  They
 *	should not try to operate on user space addresses, or to use
 *	virtual memory facilities other than those noted as interrupt
 *	safe.
 *
 *	Unlike hardware interrupts, software interrupts do have thread
 *	context.  They may block on synchronization objects, sleep, and
 *	resume execution at a later time.
 *
 *	Since software interrupts are a limited resource and run with
 *	higher priority than most other LWPs in the system, all
 *	block-and-resume activity by a software interrupt must be kept
 *	short to allow further processing at that level to continue.  By
 *	extension, code running with process context must take care to
 *	ensure that any lock that may be taken from a software interrupt
 *	can not be held for more than a short period of time.
 *
 *	The kernel does not allow software interrupts to use facilities
 *	or perform actions that may block for a significant amount of
 *	time.  This means that it's not valid for a software interrupt
 *	to sleep on condition variables	or wait for resources to become
 *	available (for example,	memory).
 *
 * Per-CPU operation
 *
 *	If a soft interrupt is triggered on a CPU, it can only be
 *	dispatched on the same CPU.  Each LWP dedicated to handling a
 *	soft interrupt is bound to its home CPU, so if the LWP blocks
 *	and needs to run again, it can only run there.  Nearly all data
 *	structures used to manage software interrupts are per-CPU.
 *
 *	The per-CPU requirement is intended to reduce "ping-pong" of
 *	cache lines between CPUs: lines occupied by data structures
 *	used to manage the soft interrupts, and lines occupied by data
 *	items being passed down to the soft interrupt.  As a positive
 *	side effect, this also means that the soft interrupt dispatch
 *	code does not need to to use spinlocks to synchronize.
 *
 * Generic implementation
 *
 *	A generic, low performance implementation is provided that
 *	works across all architectures, with no machine-dependent
 *	modifications needed.  This implementation uses the scheduler,
 *	and so has a number of restrictions:
 *
 *	1) The software interrupts are not currently preemptive, so
 *	must wait for the currently executing LWP to yield the CPU.
 *	This can introduce latency.
 *
 *	2) An expensive context switch is required for a software
 *	interrupt to be handled.
 *
 * 'Fast' software interrupts
 *
 *	If an architectures defines __HAVE_FAST_SOFTINTS, it implements
 *	the fast mechanism.  Threads running either in the kernel or in
 *	userspace will be interrupted, but will not be preempted.  When
 *	the soft interrupt completes execution, the interrupted LWP
 *	is resumed.  Interrupt dispatch code must provide the minimum
 *	level of context necessary for the soft interrupt to block and
 *	be resumed at a later time.  The machine-dependent dispatch
 *	path looks something like the following:
 *
 *	softintr()
 *	{
 *		go to IPL_HIGH if necessary for switch;
 *		save any necessary registers in a format that can be
 *		    restored by cpu_switchto if the softint blocks;
 *		arrange for cpu_switchto() to restore into the
 *		    trampoline function;
 *		identify LWP to handle this interrupt;
 *		switch to the LWP's stack;
 *		switch register stacks, if necessary;
 *		assign new value of curlwp;
 *		call MI softint_dispatch, passing old curlwp and IPL
 *		    to execute interrupt at;
 *		switch back to old stack;
 *		switch back to old register stack, if necessary;
 *		restore curlwp;
 *		return to interrupted LWP;
 *	}
 *
 *	If the soft interrupt blocks, a trampoline function is returned
 *	to in the context of the interrupted LWP, as arranged for by
 *	softint():
 *
 *	softint_ret()
 *	{
 *		unlock soft interrupt LWP;
 *		resume interrupt processing, likely returning to
 *		    interrupted LWP or dispatching another, different
 *		    interrupt;
 *	}
 *
 *	Once the soft interrupt has fired (and even if it has blocked),
 *	no further soft interrupts at that level will be triggered by
 *	MI code until the soft interrupt handler has ceased execution.
 *	If a soft interrupt handler blocks and is resumed, it resumes
 *	execution as a normal LWP (kthread) and gains VM context.  Only
 *	when it has completed and is ready to fire again will it
 *	interrupt other threads.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $");

#include <sys/param.h>
#include <sys/proc.h>
#include <sys/intr.h>
#include <sys/ipi.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/evcnt.h>
#include <sys/cpu.h>
#include <sys/xcall.h>
#include <sys/psref.h>
#include <sys/sdt.h>

#include <uvm/uvm_extern.h>

/* This could overlap with signal info in struct lwp. */
typedef struct softint {
	SIMPLEQ_HEAD(, softhand) si_q;
	struct lwp		*si_lwp;
	struct cpu_info		*si_cpu;
	uintptr_t		si_machdep;
	struct evcnt		si_evcnt;
	struct evcnt		si_evcnt_block;
	volatile int		si_active;
	int			si_ipl;
	char			si_name[8];
	char			si_name_block[8+6];
} softint_t;

typedef struct softhand {
	SIMPLEQ_ENTRY(softhand)	sh_q;
	void			(*sh_func)(void *);
	void			*sh_arg;
	softint_t		*sh_isr;
	u_int			sh_flags;
	u_int			sh_ipi_id;
} softhand_t;

typedef struct softcpu {
	struct cpu_info		*sc_cpu;
	softint_t		sc_int[SOFTINT_COUNT];
	softhand_t		sc_hand[1];
} softcpu_t;

static void	softint_thread(void *);

u_int		softint_bytes = 32768;
u_int		softint_timing;
static u_int	softint_max;
static kmutex_t	softint_lock;

SDT_PROBE_DEFINE4(sdt, kernel, softint, establish,
    "void *"/*sih*/,
    "void (*)(void *)"/*func*/,
    "void *"/*arg*/,
    "unsigned"/*flags*/);

SDT_PROBE_DEFINE1(sdt, kernel, softint, disestablish,
    "void *"/*sih*/);

SDT_PROBE_DEFINE2(sdt, kernel, softint, schedule,
    "void *"/*sih*/,
    "struct cpu_info *"/*ci*/);

SDT_PROBE_DEFINE4(sdt, kernel, softint, entry,
    "void *"/*sih*/,
    "void (*)(void *)"/*func*/,
    "void *"/*arg*/,
    "unsigned"/*flags*/);

SDT_PROBE_DEFINE4(sdt, kernel, softint, return,
    "void *"/*sih*/,
    "void (*)(void *)"/*func*/,
    "void *"/*arg*/,
    "unsigned"/*flags*/);

/*
 * softint_init_isr:
 *
 *	Initialize a single interrupt level for a single CPU.
 */
static void
softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level,
    int ipl)
{
	struct cpu_info *ci;
	softint_t *si;
	int error;

	si = &sc->sc_int[level];
	ci = sc->sc_cpu;
	si->si_cpu = ci;

	SIMPLEQ_INIT(&si->si_q);

	error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR |
	    KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp,
	    "soft%s/%u", desc, ci->ci_index);
	if (error != 0)
		panic("softint_init_isr: error %d", error);

	snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc,
	    ci->ci_index);
	evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL,
	   "softint", si->si_name);
	snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u",
	    desc, ci->ci_index);
	evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL,
	   "softint", si->si_name_block);

	si->si_ipl = ipl;
	si->si_lwp->l_private = si;
	softint_init_md(si->si_lwp, level, &si->si_machdep);
}

/*
 * softint_init:
 *
 *	Initialize per-CPU data structures.  Called from mi_cpu_attach().
 */
void
softint_init(struct cpu_info *ci)
{
	static struct cpu_info *first;
	softcpu_t *sc, *scfirst;
	softhand_t *sh, *shmax;

	if (first == NULL) {
		/* Boot CPU. */
		first = ci;
		mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE);
		softint_bytes = round_page(softint_bytes);
		softint_max = (softint_bytes - sizeof(softcpu_t)) /
		    sizeof(softhand_t);
	}

	/* Use uvm_km(9) for persistent, page-aligned allocation. */
	sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0,
	    UVM_KMF_WIRED | UVM_KMF_ZERO);
	if (sc == NULL)
		panic("softint_init_cpu: cannot allocate memory");

	ci->ci_data.cpu_softcpu = sc;
	ci->ci_data.cpu_softints = 0;
	sc->sc_cpu = ci;

	softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET,
	    IPL_SOFTNET);
	softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO,
	    IPL_SOFTBIO);
	softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK,
	    IPL_SOFTCLOCK);
	softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL,
	    IPL_SOFTSERIAL);

	if (first != ci) {
		mutex_enter(&softint_lock);
		scfirst = first->ci_data.cpu_softcpu;
		sh = sc->sc_hand;
		memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max);
		/* Update pointers for this CPU. */
		for (shmax = sh + softint_max; sh < shmax; sh++) {
			if (sh->sh_func == NULL)
				continue;
			sh->sh_isr =
			    &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK];
		}
		mutex_exit(&softint_lock);
	}
}

/*
 * softint_establish:
 *
 *	Register a software interrupt handler.
 */
void *
softint_establish(u_int flags, void (*func)(void *), void *arg)
{
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;
	softcpu_t *sc;
	softhand_t *sh;
	u_int level, index;
	u_int ipi_id = 0;
	void *sih;

	level = (flags & SOFTINT_LVLMASK);
	KASSERT(level < SOFTINT_COUNT);
	KASSERT((flags & SOFTINT_IMPMASK) == 0);

	mutex_enter(&softint_lock);

	/* Find a free slot. */
	sc = curcpu()->ci_data.cpu_softcpu;
	for (index = 1; index < softint_max; index++) {
		if (sc->sc_hand[index].sh_func == NULL)
			break;
	}
	if (index == softint_max) {
		mutex_exit(&softint_lock);
		printf("WARNING: softint_establish: table full, "
		    "increase softint_bytes\n");
		return NULL;
	}
	sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc);

	if (flags & SOFTINT_RCPU) {
		if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) {
			mutex_exit(&softint_lock);
			return NULL;
		}
	}

	/* Set up the handler on each CPU. */
	if (ncpu < 2) {
		/* XXX hack for machines with no CPU_INFO_FOREACH() early on */
		sc = curcpu()->ci_data.cpu_softcpu;
		sh = &sc->sc_hand[index];
		sh->sh_isr = &sc->sc_int[level];
		sh->sh_func = func;
		sh->sh_arg = arg;
		sh->sh_flags = flags;
		sh->sh_ipi_id = ipi_id;
	} else for (CPU_INFO_FOREACH(cii, ci)) {
		sc = ci->ci_data.cpu_softcpu;
		sh = &sc->sc_hand[index];
		sh->sh_isr = &sc->sc_int[level];
		sh->sh_func = func;
		sh->sh_arg = arg;
		sh->sh_flags = flags;
		sh->sh_ipi_id = ipi_id;
	}
	mutex_exit(&softint_lock);

	SDT_PROBE4(sdt, kernel, softint, establish,  sih, func, arg, flags);

	return sih;
}

/*
 * softint_disestablish:
 *
 *	Unregister a software interrupt handler.  The soft interrupt could
 *	still be active at this point, but the caller commits not to try
 *	and trigger it again once this call is made.  The caller must not
 *	hold any locks that could be taken from soft interrupt context,
 *	because we will wait for the softint to complete if it's still
 *	running.
 */
void
softint_disestablish(void *arg)
{
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;
	softcpu_t *sc;
	softhand_t *sh;
	uintptr_t offset;

	offset = (uintptr_t)arg;
	KASSERT(offset != 0);
	KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u",
	    offset, softint_bytes);

	/*
	 * Unregister IPI handler if there is any.  Note: there is no need
	 * to disable preemption here - ID is stable.
	 */
	sc = curcpu()->ci_data.cpu_softcpu;
	sh = (softhand_t *)((uint8_t *)sc + offset);
	if (sh->sh_ipi_id) {
		ipi_unregister(sh->sh_ipi_id);
	}

	/*
	 * Run a dummy softint at the same level on all CPUs and wait for
	 * completion, to make sure this softint is no longer running
	 * anywhere.
	 */
	xc_barrier(XC_HIGHPRI_IPL(sh->sh_isr->si_ipl));

	/*
	 * Notify dtrace probe when the old softint can't be running
	 * any more, but before it can be recycled for a new softint.
	 */
	SDT_PROBE1(sdt, kernel, softint, disestablish,  arg);

	/* Clear the handler on each CPU. */
	mutex_enter(&softint_lock);
	for (CPU_INFO_FOREACH(cii, ci)) {
		sc = ci->ci_data.cpu_softcpu;
		sh = (softhand_t *)((uint8_t *)sc + offset);
		KASSERT(sh->sh_func != NULL);
		sh->sh_func = NULL;
	}
	mutex_exit(&softint_lock);
}

/*
 * softint_schedule:
 *
 *	Trigger a software interrupt.  Must be called from a hardware
 *	interrupt handler, or with preemption disabled (since we are
 *	using the value of curcpu()).
 */
void
softint_schedule(void *arg)
{
	softhand_t *sh;
	softint_t *si;
	uintptr_t offset;
	int s;

	SDT_PROBE2(sdt, kernel, softint, schedule,  arg, /*ci*/NULL);

	/*
	 * If this assert fires, rather than disabling preemption explicitly
	 * to make it stop, consider that you are probably using a softint
	 * when you don't need to.
	 */
	KASSERT(kpreempt_disabled());

	/* Find the handler record for this CPU. */
	offset = (uintptr_t)arg;
	KASSERT(offset != 0);
	KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u",
	    offset, softint_bytes);
	sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset);

	/* If it's already pending there's nothing to do. */
	if ((sh->sh_flags & SOFTINT_PENDING) != 0) {
		return;
	}

	/*
	 * Enqueue the handler into the LWP's pending list.
	 * If the LWP is completely idle, then make it run.
	 */
	s = splhigh();
	if ((sh->sh_flags & SOFTINT_PENDING) == 0) {
		si = sh->sh_isr;
		sh->sh_flags |= SOFTINT_PENDING;
		SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q);
		if (si->si_active == 0) {
			si->si_active = 1;
			softint_trigger(si->si_machdep);
		}
	}
	splx(s);
}

/*
 * softint_schedule_cpu:
 *
 *	Trigger a software interrupt on a target CPU.  This invokes
 *	softint_schedule() for the local CPU or send an IPI to invoke
 *	this routine on the remote CPU.  Preemption must be disabled.
 */
void
softint_schedule_cpu(void *arg, struct cpu_info *ci)
{
	KASSERT(kpreempt_disabled());

	if (curcpu() != ci) {
		const softcpu_t *sc = ci->ci_data.cpu_softcpu;
		const uintptr_t offset = (uintptr_t)arg;
		const softhand_t *sh;

		SDT_PROBE2(sdt, kernel, softint, schedule,  arg, ci);
		sh = (const softhand_t *)((const uint8_t *)sc + offset);
		KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0);
		ipi_trigger(sh->sh_ipi_id, ci);
		return;
	}

	/* Just a local CPU. */
	softint_schedule(arg);
}

/*
 * softint_execute:
 *
 *	Invoke handlers for the specified soft interrupt.
 *	Must be entered at splhigh.  Will drop the priority
 *	to the level specified, but returns back at splhigh.
 */
static inline void
softint_execute(lwp_t *l, int s)
{
	softint_t *si = l->l_private;
	softhand_t *sh;

	KASSERT(si->si_lwp == curlwp);
	KASSERT(si->si_cpu == curcpu());
	KASSERT(si->si_lwp->l_wchan == NULL);
	KASSERT(si->si_active);
	KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d",
	    l, l->l_nopreempt);

	/*
	 * Note: due to priority inheritance we may have interrupted a
	 * higher priority LWP.  Since the soft interrupt must be quick
	 * and is non-preemptable, we don't bother yielding.
	 */

	while (!SIMPLEQ_EMPTY(&si->si_q)) {
		/*
		 * Pick the longest waiting handler to run.  We block
		 * interrupts but do not lock in order to do this, as
		 * we are protecting against the local CPU only.
		 */
		sh = SIMPLEQ_FIRST(&si->si_q);
		SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q);
		KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0);
		sh->sh_flags ^= SOFTINT_PENDING;
		splx(s);

		/* Run the handler. */
		SDT_PROBE4(sdt, kernel, softint, entry,
		    ((const char *)sh -
			(const char *)curcpu()->ci_data.cpu_softcpu),
		    sh->sh_func, sh->sh_arg, sh->sh_flags);
		if (__predict_true((sh->sh_flags & SOFTINT_MPSAFE) != 0)) {
			(*sh->sh_func)(sh->sh_arg);
		} else {
			KERNEL_LOCK(1, l);
			(*sh->sh_func)(sh->sh_arg);
			KERNEL_UNLOCK_ONE(l);
		}
		SDT_PROBE4(sdt, kernel, softint, return,
		    ((const char *)sh -
			(const char *)curcpu()->ci_data.cpu_softcpu),
		    sh->sh_func, sh->sh_arg, sh->sh_flags);

		/* Diagnostic: check that spin-locks have not leaked. */
		KASSERTMSG(curcpu()->ci_mtx_count == 0,
		    "%s: ci_mtx_count (%d) != 0, sh_func %p\n",
		    __func__, curcpu()->ci_mtx_count, sh->sh_func);
		/* Diagnostic: check that psrefs have not leaked. */
		KASSERTMSG(l->l_psrefs == 0, "%s: l_psrefs=%d, sh_func=%p\n",
		    __func__, l->l_psrefs, sh->sh_func);
		/* Diagnostic: check that biglocks have not leaked. */
		KASSERTMSG(l->l_blcnt == 0,
		    "%s: sh_func=%p leaked %d biglocks",
		    __func__, sh->sh_func, curlwp->l_blcnt);
		/* Diagnostic: check that LWP nopreempt remains zero. */
		KASSERTMSG(l->l_nopreempt == 0,
		    "%s: lwp %p nopreempt %d func %p",
		    __func__, l, l->l_nopreempt, sh->sh_func);

		(void)splhigh();
	}

	PSREF_DEBUG_BARRIER();

	CPU_COUNT(CPU_COUNT_NSOFT, 1);

	KASSERT(si->si_cpu == curcpu());
	KASSERT(si->si_lwp->l_wchan == NULL);
	KASSERT(si->si_active);
	si->si_evcnt.ev_count++;
	si->si_active = 0;
}

/*
 * softint_block:
 *
 *	Update statistics when the soft interrupt blocks.
 */
void
softint_block(lwp_t *l)
{
	softint_t *si = l->l_private;

	KASSERT((l->l_pflag & LP_INTR) != 0);
	si->si_evcnt_block.ev_count++;
}

#ifndef __HAVE_FAST_SOFTINTS

#ifdef __HAVE_PREEMPTION
#error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS
#endif

/*
 * softint_init_md:
 *
 *	Slow path: perform machine-dependent initialization.
 */
void
softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
{
	struct proc *p;
	softint_t *si;

	*machdep = (1 << level);
	si = l->l_private;
	p = l->l_proc;

	mutex_enter(p->p_lock);
	lwp_lock(l);
	/* Cheat and make the KASSERT in softint_thread() happy. */
	si->si_active = 1;
	setrunnable(l);
	/* LWP now unlocked */
	mutex_exit(p->p_lock);
}

/*
 * softint_trigger:
 *
 *	Slow path: cause a soft interrupt handler to begin executing.
 *	Called at IPL_HIGH.
 */
void
softint_trigger(uintptr_t machdep)
{
	struct cpu_info *ci;
	lwp_t *l;

	ci = curcpu();
	ci->ci_data.cpu_softints |= machdep;
	l = ci->ci_onproc;

	/*
	 * Arrange for mi_switch() to be called.  If called from interrupt
	 * mode, we don't know if curlwp is executing in kernel or user, so
	 * post an AST and have it take a trip through userret().  If not in
	 * interrupt mode, curlwp is running in kernel and will notice the
	 * resched soon enough; avoid the AST.
	 */
	if (l == ci->ci_data.cpu_idlelwp) {
		atomic_or_uint(&ci->ci_want_resched,
		    RESCHED_IDLE | RESCHED_UPREEMPT);
	} else {
		atomic_or_uint(&ci->ci_want_resched, RESCHED_UPREEMPT);
		if (cpu_intr_p()) {
			cpu_signotify(l);
		}
	}
}

/*
 * softint_thread:
 *
 *	Slow path: MI software interrupt dispatch.
 */
void
softint_thread(void *cookie)
{
	softint_t *si;
	lwp_t *l;
	int s;

	l = curlwp;
	si = l->l_private;

	for (;;) {
		/* Clear pending status and run it. */
		s = splhigh();
		l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep;
		softint_execute(l, s);
		splx(s);

		/* Interrupts allowed to run again before switching. */
		lwp_lock(l);
		l->l_stat = LSIDL;
		spc_lock(l->l_cpu);
		mi_switch(l);
	}
}

/*
 * softint_picklwp:
 *
 *	Slow path: called from mi_switch() to pick the highest priority
 *	soft interrupt LWP that needs to run.
 */
lwp_t *
softint_picklwp(void)
{
	struct cpu_info *ci;
	u_int mask;
	softint_t *si;
	lwp_t *l;

	ci = curcpu();
	si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
	mask = ci->ci_data.cpu_softints;

	if ((mask & (1 << SOFTINT_SERIAL)) != 0) {
		l = si[SOFTINT_SERIAL].si_lwp;
	} else if ((mask & (1 << SOFTINT_NET)) != 0) {
		l = si[SOFTINT_NET].si_lwp;
	} else if ((mask & (1 << SOFTINT_BIO)) != 0) {
		l = si[SOFTINT_BIO].si_lwp;
	} else if ((mask & (1 << SOFTINT_CLOCK)) != 0) {
		l = si[SOFTINT_CLOCK].si_lwp;
	} else {
		panic("softint_picklwp");
	}

	return l;
}

#else	/*  !__HAVE_FAST_SOFTINTS */

/*
 * softint_thread:
 *
 *	Fast path: the LWP is switched to without restoring any state,
 *	so we should not arrive here - there is a direct handoff between
 *	the interrupt stub and softint_dispatch().
 */
void
softint_thread(void *cookie)
{

	panic("softint_thread");
}

/*
 * softint_dispatch:
 *
 *	Fast path: entry point from machine-dependent code.
 */
void
softint_dispatch(lwp_t *pinned, int s)
{
	struct bintime now;
	u_int timing;
	lwp_t *l;

#ifdef DIAGNOSTIC
	if ((pinned->l_pflag & LP_RUNNING) == 0 || curlwp->l_stat != LSIDL) {
		struct lwp *onproc = curcpu()->ci_onproc;
		int s2 = splhigh();
		printf("curcpu=%d, spl=%d curspl=%d\n"
			"onproc=%p => l_stat=%d l_flag=%08x l_cpu=%d\n"
			"curlwp=%p => l_stat=%d l_flag=%08x l_cpu=%d\n"
			"pinned=%p => l_stat=%d l_flag=%08x l_cpu=%d\n",
			cpu_index(curcpu()), s, s2, onproc, onproc->l_stat,
			onproc->l_flag, cpu_index(onproc->l_cpu), curlwp,
			curlwp->l_stat, curlwp->l_flag,
			cpu_index(curlwp->l_cpu), pinned, pinned->l_stat,
			pinned->l_flag, cpu_index(pinned->l_cpu));
		splx(s2);
		panic("softint screwup");
	}
#endif

	/*
	 * Note the interrupted LWP, and mark the current LWP as running
	 * before proceeding.  Although this must as a rule be done with
	 * the LWP locked, at this point no external agents will want to
	 * modify the interrupt LWP's state.
	 */
	timing = softint_timing;
	l = curlwp;
	l->l_switchto = pinned;
	l->l_stat = LSONPROC;

	/*
	 * Dispatch the interrupt.  If softints are being timed, charge
	 * for it.
	 */
	if (timing) {
		binuptime(&l->l_stime);
		membar_producer();	/* for calcru */
		l->l_pflag |= LP_TIMEINTR;
	}
	l->l_pflag |= LP_RUNNING;
	softint_execute(l, s);
	if (timing) {
		binuptime(&now);
		updatertime(l, &now);
		l->l_pflag &= ~LP_TIMEINTR;
	}

	/*
	 * If we blocked while handling the interrupt, the pinned LWP is
	 * gone and we are now running as a kthread, so find another LWP to
	 * run.  softint_dispatch() won't be reentered until the priority is
	 * finally dropped to IPL_NONE on entry to the next LWP on this CPU.
	 */
	l->l_stat = LSIDL;
	if (l->l_switchto == NULL) {
		lwp_lock(l);
		spc_lock(l->l_cpu);
		mi_switch(l);
		/* NOTREACHED */
	}
	l->l_switchto = NULL;
	l->l_pflag &= ~LP_RUNNING;
}

#endif	/* !__HAVE_FAST_SOFTINTS */