MI IPI interface:

- Implement support for the asynchronous IPI calls.
- Rework synchronous IPI code to reuse the asynchronous mechanism.
- Add ipi(9) manual page; needs wizd(8).

Note: MD code can now provide a low level primitive for the ipi(9) and
reuse this interface instead of open-coding.  Portmasters are encouraged
to convert.  Ride 6.99.43!
This commit is contained in:
rmind 2014-05-25 15:34:19 +00:00
parent b8e866a463
commit 3da69dd68c
6 changed files with 355 additions and 17 deletions

View File

@ -1,4 +1,4 @@
# $NetBSD: mi,v 1.1892 2014/05/15 16:32:28 apb Exp $
# $NetBSD: mi,v 1.1893 2014/05/25 15:34:19 rmind Exp $
#
# Note: don't delete entries from here - mark them as "obsolete" instead.
#
@ -10273,6 +10273,7 @@
./usr/share/man/cat9/ioctl.0 comp-sys-catman .cat
./usr/share/man/cat9/ioctl_copyin.0 comp-sys-catman .cat
./usr/share/man/cat9/ioctl_copyout.0 comp-sys-catman .cat
./usr/share/man/cat9/ipi.0 comp-sys-catman .cat
./usr/share/man/cat9/ipkdb.0 comp-sys-catman .cat
./usr/share/man/cat9/ipkdb_connect.0 comp-sys-catman .cat
./usr/share/man/cat9/ipkdb_init.0 comp-sys-catman .cat
@ -16970,6 +16971,7 @@
./usr/share/man/html9/ioctl.html comp-sys-htmlman html
./usr/share/man/html9/ioctl_copyin.html comp-sys-htmlman html
./usr/share/man/html9/ioctl_copyout.html comp-sys-htmlman html
./usr/share/man/html9/ipi.html comp-sys-htmlman html
./usr/share/man/html9/ipkdb.html comp-sys-htmlman html
./usr/share/man/html9/ipkdb_connect.html comp-sys-htmlman html
./usr/share/man/html9/ipkdb_init.html comp-sys-htmlman html
@ -23820,6 +23822,7 @@
./usr/share/man/man9/ioctl.9 comp-sys-man .man
./usr/share/man/man9/ioctl_copyin.9 comp-sys-man .man
./usr/share/man/man9/ioctl_copyout.9 comp-sys-man .man
./usr/share/man/man9/ipi.9 comp-sys-man .man
./usr/share/man/man9/ipkdb.9 comp-sys-man .man
./usr/share/man/man9/ipkdb_connect.9 comp-sys-man .man
./usr/share/man/man9/ipkdb_init.9 comp-sys-man .man

View File

@ -1,4 +1,4 @@
# $NetBSD: Makefile,v 1.377 2014/03/24 13:42:40 hannken Exp $
# $NetBSD: Makefile,v 1.378 2014/05/25 15:34:20 rmind Exp $
# Makefile for section 9 (kernel function and variable) manual pages.
@ -25,7 +25,7 @@ MAN= accept_filter.9 accf_data.9 accf_http.9 \
ieee80211_node.9 ieee80211_output.9 ieee80211_proto.9 \
ieee80211_radiotap.9 iic.9 imax.9 \
in_getifa.9 \
in4_cksum.9 inittodr.9 intro.9 ioasic.9 ioctl.9 ipkdb.9 isa.9 \
in4_cksum.9 inittodr.9 intro.9 ioasic.9 ioctl.9 ipkdb.9 ipi.9 isa.9 \
isapnp.9 itimerfix.9 kauth.9 kcopy.9 kcpuset.9 kmem.9 \
kpause.9 \
kfilter_register.9 knote.9 \

177
share/man/man9/ipi.9 Normal file
View File

@ -0,0 +1,177 @@
.\" $NetBSD: ipi.9,v 1.1 2014/05/25 15:34:20 rmind Exp $
.\"
.\" Copyright (c) 2014 The NetBSD Foundation, Inc.
.\" All rights reserved.
.\"
.\" This code is derived from software contributed to The NetBSD Foundation
.\" by Mindaugas Rasiukevicius.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd May 25, 2014
.Dt IPI 9
.Os
.Sh NAME
.Nm ipi
.Nd MI IPI interface
.Sh SYNOPSIS
.In sys/ipi.h
.Vt typedef void (*ipi_func_t)(void *);
.\" -----
.Ft u_int
.Fn ipi_register "ipi_func_t func" "void *arg"
.Ft void
.Fn ipi_unregister "u_int ipi_id"
.Ft void
.Fn ipi_trigger "u_int ipi_id" "struct cpu_info *ci"
.\" -----
.Ft void
.Fn ipi_unicast "ipi_msg_t *msg" "struct cpu_info *ci"
.Ft void
.Fn ipi_multicast "ipi_msg_t *msg" "const kcpuset_t *target"
.Ft void
.Fn ipi_broadcast "ipi_msg_t *msg"
.Ft void
.Fn ipi_wait "ipi_msg_t *msg"
.\" -----
.Sh DESCRIPTION
The machine-independent
.Nm
interface provides capability to send inter-processor interrupts (IPIs)
amongst CPUs.
The interface has two mechanisms: asynchronous IPI to invoke functions
with a constant argument and synchronous IPIs with the cross-call support.
.Pp
Other synchronization interfaces are built using the MI IPI interface.
For a general purpose inter-processor cross-calls or remote interrupts, use
.Xr xcall 9
or
.Xr softint 9
interfaces.
.Pp
The primary use cases of the MI IPIs include the following:
.Bl -hyphen -compact
.It
provide a facility for
.Xr softint 9
subsystem to schedule software interrupts on the remote CPUs
.It
provide a facility for
.Xr xcall 9
subsystem
.It
abstract IPI handling and facilitate the machine-dependent code
.El
.Pp
.\" -----
.Ss Asynchronous IPI interface
This interface allows dynamic registration of IPI handlers with a constant
argument and asynchronous triggering of the interrupts.
.Bl -tag -width compact
.It Fn ipi_register "func" "arg"
Register an IPI handler
.Fa func
with an arbitrary argument
.Fa arg .
Returns non-zero IPI identifier on success and zero on failure.
.It Fn ipi_unregister "ipi_id"
Unregister the IPI handler identified by the
.Fa ipi_id .
.It Fn ipi_trigger "ipi_id" "ci"
Trigger an IPI identified by
.Fa ipi_id
on a remote CPU specified by
.Fa ci .
This function must be called with the kernel preemption disabled and
the target CPU must be remote.
.El
.Pp
.\" -----
.Ss Synchronous IPI interface
This interface provides capability to perform cross-calls, i.e. invoke
an arbitrary function on a remote CPU.
The invocations are performed synchronously and the caller must wait
for completion.
The cross-call is described by an IPI "message".
The caller has to fill
.Vt ipi_msg_t
structure which has the following public members:
.Bd -literal
ipi_func_t func;
void arg;
.Ed
.Pp
The
.Ar func
member specifies a function to invoke and
.Ar arg
is the argument to be passed to the function.
.Pp
.Bl -tag -width compact
.It Fn ipi_unicast "msg" "ci"
Send an IPI to a remote CPU specified by
.Fa ci .
.It Fn ipi_multicast "msg" "target"
Send IPIs to a CPU set specified by the
.Fa target .
.It Fn ipi_broadcast "msg"
Send IPIs to all CPUs.
.It Fn ipi_wait "msg"
Wait until all IPIs complete.
.El
.Pp
All described functions, except
.Fn ipi_wait ,
must be called with the kernel preemption disabled.
All synchronous IPI invocations must be awaited for completion with
.Fn ipi_wait
function, before IPI message structure can be destroyed or new
cross-call requests performed.
.\" -----
.Sh NOTES
Functions being called must be lightweight.
They run at
.Em IPL_HIGH
and should generally not use any other synchronization interfaces,
such as
.Xr mutex 9 .
If spin-locks are used, they must be used carefully and have no contention.
.\" -----
.Sh CODE REFERENCES
The
.Nm
interface is implemented within the file
.Pa sys/kern/subr_ipi.c .
.\" -----
.Sh SEE ALSO
.Xr softint 9 ,
.Xr spl 9 ,
.Xr kcpuset 9 ,
.Xr kpreempt 9 ,
.Xr xcall 9
.Sh HISTORY
The
.Nm
interface first appeared in
.Nx 7.0 .
.Sh AUTHORS
.An Mindaugas Rasiukevicius Aq Mt rmind@NetBSD.org

View File

@ -1,4 +1,4 @@
/* $NetBSD: subr_ipi.c,v 1.1 2014/05/19 22:47:54 rmind Exp $ */
/* $NetBSD: subr_ipi.c,v 1.2 2014/05/25 15:34:19 rmind Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -30,11 +30,13 @@
*/
/*
* Inter-processor interrupt (IPI) interface with cross-call support.
* Inter-processor interrupt (IPI) interface: asynchronous IPIs to
* invoke functions with a constant argument and synchronous IPIs
* with the cross-call support.
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.1 2014/05/19 22:47:54 rmind Exp $");
__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.2 2014/05/25 15:34:19 rmind Exp $");
#include <sys/param.h>
#include <sys/types.h>
@ -46,10 +48,25 @@ __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.1 2014/05/19 22:47:54 rmind Exp $");
#include <sys/kcpuset.h>
#include <sys/kmem.h>
#include <sys/lock.h>
#include <sys/mutex.h>
/*
* An array of the IPI handlers used for asynchronous invocation.
* The lock protects the slot allocation.
*/
typedef struct {
ipi_func_t func;
void * arg;
} ipi_intr_t;
static kmutex_t ipi_mngmt_lock;
static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned;
/*
* Per-CPU mailbox for IPI messages: it is a single cache line storing
* up to IPI_MSG_MAX messages.
* up to IPI_MSG_MAX messages. This interface is built on top of the
* synchronous IPIs.
*/
#define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
@ -59,8 +76,14 @@ typedef struct {
ipi_msg_t * msg[IPI_MSG_SLOTS];
} ipi_mbox_t;
/* Mailboxes for the synchronous IPIs. */
static ipi_mbox_t * ipi_mboxes __read_mostly;
static struct evcnt ipi_mboxfull_ev __cacheline_aligned;
static void ipi_msg_cpu_handler(void *);
/* Handler for the synchronous IPIs - it must be zero. */
#define IPI_SYNCH_ID 0
#ifndef MULTIPROCESSOR
#define cpu_ipi(ci) KASSERT(ci == NULL)
@ -71,14 +94,99 @@ ipi_sysinit(void)
{
const size_t len = ncpu * sizeof(ipi_mbox_t);
/* Initialise the per-CPU bit fields. */
for (u_int i = 0; i < ncpu; i++) {
struct cpu_info *ci = cpu_lookup(i);
memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
}
mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
memset(ipi_intrs, 0, sizeof(ipi_intrs));
/* Allocate per-CPU IPI mailboxes. */
ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
KASSERT(ipi_mboxes != NULL);
/*
* Register the handler for synchronous IPIs. This mechanism
* is built on top of the asynchronous interface. Slot zero is
* reserved permanently; it is also handy to use zero as a failure
* for other registers (as it is potentially less error-prone).
*/
ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;
evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
"ipi", "full");
}
/*
* ipi_register: register an asynchronous IPI handler.
*
* => Returns IPI ID which is greater than zero; on failure - zero.
*/
u_int
ipi_register(ipi_func_t func, void *arg)
{
mutex_enter(&ipi_mngmt_lock);
for (u_int i = 0; i < IPI_MAXREG; i++) {
if (ipi_intrs[i].func == NULL) {
/* Register the function. */
ipi_intrs[i].func = func;
ipi_intrs[i].arg = arg;
mutex_exit(&ipi_mngmt_lock);
KASSERT(i != IPI_SYNCH_ID);
return i;
}
}
mutex_exit(&ipi_mngmt_lock);
printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
return 0;
}
/*
* ipi_unregister: release the IPI handler given the ID.
*/
void
ipi_unregister(u_int ipi_id)
{
ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop };
KASSERT(ipi_id != IPI_SYNCH_ID);
KASSERT(ipi_id < IPI_MAXREG);
/* Release the slot. */
mutex_enter(&ipi_mngmt_lock);
KASSERT(ipi_intrs[ipi_id].func != NULL);
ipi_intrs[ipi_id].func = NULL;
/* Ensure that there are no IPIs in flight. */
kpreempt_disable();
ipi_broadcast(&ipimsg);
ipi_wait(&ipimsg);
kpreempt_enable();
mutex_exit(&ipi_mngmt_lock);
}
/*
* ipi_trigger: asynchronously send an IPI to the specified CPU.
*/
void
ipi_trigger(u_int ipi_id, struct cpu_info *ci)
{
const u_int i = ipi_id >> IPI_BITW_SHIFT;
const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);
KASSERT(ipi_id < IPI_MAXREG);
KASSERT(kpreempt_disabled());
KASSERT(curcpu() != ci);
/* Mark as pending and send an IPI. */
if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) {
atomic_or_32(&ci->ci_ipipend[i], bitm);
cpu_ipi(ci);
}
}
/*
* put_msg: insert message into the mailbox.
*/
@ -105,12 +213,44 @@ again:
*/
void
ipi_cpu_handler(void)
{
struct cpu_info * const ci = curcpu();
/*
* Handle asynchronous IPIs: inspect per-CPU bit field, extract
* IPI ID numbers and execute functions in those slots.
*/
for (u_int i = 0; i < IPI_BITWORDS; i++) {
uint32_t pending, bit;
if (ci->ci_ipipend[i] == 0) {
continue;
}
pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_producer();
#endif
while ((bit = ffs(pending)) != 0) {
const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];
pending &= ~(1U << bit);
KASSERT(ipi_hdl->func != NULL);
ipi_hdl->func(ipi_hdl->arg);
}
}
}
/*
* ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
* execute the passed functions and acknowledge the messages.
*/
static void
ipi_msg_cpu_handler(void *arg __unused)
{
const struct cpu_info * const ci = curcpu();
ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];
KASSERT(curcpu() == ci);
for (u_int i = 0; i < IPI_MSG_MAX; i++) {
ipi_msg_t *msg;
@ -148,7 +288,7 @@ ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
membar_producer();
put_msg(&ipi_mboxes[id], msg);
cpu_ipi(ci);
ipi_trigger(IPI_SYNCH_ID, ci);
}
/*
@ -182,7 +322,7 @@ ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
continue;
}
put_msg(&ipi_mboxes[id], msg);
cpu_ipi(ci);
ipi_trigger(IPI_SYNCH_ID, ci);
}
if (local) {
msg->func(msg->arg);
@ -216,8 +356,8 @@ ipi_broadcast(ipi_msg_t *msg)
}
id = cpu_index(ci);
put_msg(&ipi_mboxes[id], msg);
ipi_trigger(IPI_SYNCH_ID, ci);
}
cpu_ipi(NULL);
/* Finally, execute locally. */
msg->func(msg->arg);

View File

@ -1,4 +1,4 @@
/* $NetBSD: cpu_data.h,v 1.36 2013/11/25 03:02:30 christos Exp $ */
/* $NetBSD: cpu_data.h,v 1.37 2014/05/25 15:34:19 rmind Exp $ */
/*-
* Copyright (c) 2004, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@ -43,6 +43,7 @@ struct lwp;
#include <sys/percpu_types.h>
#include <sys/queue.h>
#include <sys/kcpuset.h>
#include <sys/ipi.h>
/*
* MI per-cpu data
@ -71,6 +72,7 @@ struct cpu_data {
kcondvar_t cpu_xcall; /* cross-call support */
int cpu_xcall_pending; /* cross-call support */
lwp_t *cpu_onproc; /* bottom level LWP */
uint32_t cpu_ipipend[IPI_BITWORDS]; /* pending IPIs */
cpuid_t cpu_package_id;
cpuid_t cpu_core_id;
@ -123,6 +125,7 @@ struct cpu_data {
#define ci_lkdebug_recurse ci_data.cpu_lkdebug_recurse
#define ci_pcu_curlwp ci_data.cpu_pcu_curlwp
#define ci_kcpuset ci_data.cpu_kcpuset
#define ci_ipipend ci_data.cpu_ipipend
#define ci_package_id ci_data.cpu_package_id
#define ci_core_id ci_data.cpu_core_id

View File

@ -1,4 +1,4 @@
/* $NetBSD: ipi.h,v 1.1 2014/05/19 22:47:54 rmind Exp $ */
/* $NetBSD: ipi.h,v 1.2 2014/05/25 15:34:19 rmind Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -47,13 +47,28 @@ typedef struct {
volatile u_int _pending;
} ipi_msg_t;
/*
* Internal constants and implementation hooks.
*
* IPI_MAXREG: the maximum number of asynchronous handlers which can
* be registered on the system (normally, this should not be high).
*/
#define IPI_MAXREG 32
#define IPI_BITW_SHIFT 5
#define IPI_BITW_MASK (32 - 1)
#define IPI_BITWORDS (IPI_MAXREG >> IPI_BITW_SHIFT)
void ipi_sysinit(void);
void ipi_cpu_handler(void);
void cpu_ipi(struct cpu_info *);
/*
* Public ipi(9) API.
*/
/* Public interface: asynchronous IPIs. */
u_int ipi_register(ipi_func_t, void *);
void ipi_unregister(u_int);
void ipi_trigger(u_int, struct cpu_info *);
/* Public interface: synchronous IPIs. */
void ipi_unicast(ipi_msg_t *, struct cpu_info *);
void ipi_multicast(ipi_msg_t *, const kcpuset_t *);
void ipi_broadcast(ipi_msg_t *);