Implementation of per-CPU work-queues support for workqueue(9) interface.
WQ_PERCPU flag for workqueue and additional argument for workqueue_enqueue() to assign a CPU might be used. Notes: - For now, the list is used for workqueue_queue, which is non-optimal, and will be changed with array, where index would be CPU ID. - The data structures should be changed to be cache-friendly. Reviewed by: <yamt>, <tech-kern>
This commit is contained in:
rmind
parent
c73835c125
commit
20bbb87e34
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: dmover_io.c,v 1.26 2007/03/12 18:18:30 ad Exp $ */
|
||||
/* $NetBSD: dmover_io.c,v 1.27 2007/07/12 20:39:56 rmind Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2002, 2003 Wasabi Systems, Inc.
|
||||
|
|
@ -55,7 +55,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: dmover_io.c,v 1.26 2007/03/12 18:18:30 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: dmover_io.c,v 1.27 2007/07/12 20:39:56 rmind Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/queue.h>
|
||||
|
|
@ -320,7 +320,7 @@ dmio_usrreq_fini(struct dmio_state *ds, struct dmio_usrreq_state *dus)
|
|||
free(dus->dus_uio_in, M_TEMP);
|
||||
}
|
||||
|
||||
workqueue_enqueue(dmio_cleaner, &dus->dus_work);
|
||||
workqueue_enqueue(dmio_cleaner, &dus->dus_work, NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: sysmon_envsys_events.c,v 1.7 2007/07/10 06:32:08 yamt Exp $ */
|
||||
/* $NetBSD: sysmon_envsys_events.c,v 1.8 2007/07/12 20:39:56 rmind Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2007 The NetBSD Foundation, Inc.
|
||||
|
|
@ -41,7 +41,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: sysmon_envsys_events.c,v 1.7 2007/07/10 06:32:08 yamt Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: sysmon_envsys_events.c,v 1.8 2007/07/12 20:39:56 rmind Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/types.h>
|
||||
|
|
@ -452,7 +452,7 @@ sme_events_check(void *arg)
|
|||
see->pes.pes_dvname,
|
||||
see->pes.pes_sensname,
|
||||
see->type));
|
||||
workqueue_enqueue(seewq, &see->see_wk);
|
||||
workqueue_enqueue(seewq, &see->see_wk, NULL);
|
||||
}
|
||||
callout_schedule(&seeco, SME_EVTIMO);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: if_aue.c,v 1.101 2007/03/13 13:51:54 drochner Exp $ */
|
||||
/* $NetBSD: if_aue.c,v 1.102 2007/07/12 20:39:56 rmind Exp $ */
|
||||
/*
|
||||
* Copyright (c) 1997, 1998, 1999, 2000
|
||||
* Bill Paul <wpaul@ee.columbia.edu>. All rights reserved.
|
||||
|
|
@ -77,7 +77,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: if_aue.c,v 1.101 2007/03/13 13:51:54 drochner Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: if_aue.c,v 1.102 2007/07/12 20:39:56 rmind Exp $");
|
||||
|
||||
#if defined(__NetBSD__)
|
||||
#include "opt_inet.h"
|
||||
|
|
@ -1619,7 +1619,7 @@ aue_ioctl(struct ifnet *ifp, u_long command, void *data)
|
|||
if (error == ENETRESET) {
|
||||
if (ifp->if_flags & IFF_RUNNING) {
|
||||
#if defined(__NetBSD__)
|
||||
workqueue_enqueue(sc->wqp,&sc->wk);
|
||||
workqueue_enqueue(sc->wqp,&sc->wk, NULL);
|
||||
/* XXX */
|
||||
#else
|
||||
aue_init(sc);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: kern_physio.c,v 1.81 2007/07/09 21:10:53 ad Exp $ */
|
||||
/* $NetBSD: kern_physio.c,v 1.82 2007/07/12 20:39:56 rmind Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1982, 1986, 1990, 1993
|
||||
|
|
@ -71,7 +71,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.81 2007/07/09 21:10:53 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.82 2007/07/12 20:39:56 rmind Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
|
|
@ -223,7 +223,7 @@ physio_biodone(struct buf *bp)
|
|||
KASSERT(bp->b_resid <= bp->b_bcount);
|
||||
#endif /* defined(DIAGNOSTIC) */
|
||||
|
||||
workqueue_enqueue(physio_workqueue, &bp->b_work);
|
||||
workqueue_enqueue(physio_workqueue, &bp->b_work, NULL);
|
||||
}
|
||||
|
||||
static int
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: subr_vmem.c,v 1.31 2007/07/09 21:10:55 ad Exp $ */
|
||||
/* $NetBSD: subr_vmem.c,v 1.32 2007/07/12 20:39:56 rmind Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c)2006 YAMAMOTO Takashi,
|
||||
|
|
@ -38,7 +38,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.31 2007/07/09 21:10:55 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.32 2007/07/12 20:39:56 rmind Exp $");
|
||||
|
||||
#define VMEM_DEBUG
|
||||
#if defined(_KERNEL)
|
||||
|
|
@ -1200,7 +1200,7 @@ static void
|
|||
vmem_rehash_all_kick(void *dummy)
|
||||
{
|
||||
|
||||
workqueue_enqueue(vmem_rehash_wq, &vmem_rehash_wk);
|
||||
workqueue_enqueue(vmem_rehash_wq, &vmem_rehash_wk, NULL);
|
||||
}
|
||||
|
||||
void
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: subr_workqueue.c,v 1.13 2007/07/09 21:10:55 ad Exp $ */
|
||||
/* $NetBSD: subr_workqueue.c,v 1.14 2007/07/12 20:39:56 rmind Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c)2002, 2005 YAMAMOTO Takashi,
|
||||
|
|
@ -27,7 +27,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: subr_workqueue.c,v 1.13 2007/07/09 21:10:55 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: subr_workqueue.c,v 1.14 2007/07/12 20:39:56 rmind Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
|
|
@ -45,11 +45,12 @@ struct workqueue_queue {
|
|||
kcondvar_t q_cv;
|
||||
struct workqhead q_queue;
|
||||
struct lwp *q_worker;
|
||||
struct cpu_info *q_ci;
|
||||
SLIST_ENTRY(workqueue_queue) q_list;
|
||||
};
|
||||
|
||||
struct workqueue {
|
||||
struct workqueue_queue wq_queue; /* todo: make this per-cpu */
|
||||
|
||||
SLIST_HEAD(, workqueue_queue) wq_queue;
|
||||
void (*wq_func)(struct work *, void *);
|
||||
void *wq_arg;
|
||||
const char *wq_name;
|
||||
|
|
@ -59,6 +60,18 @@ struct workqueue {
|
|||
|
||||
#define POISON 0xaabbccdd
|
||||
|
||||
static struct workqueue_queue *
|
||||
workqueue_queue_lookup(struct workqueue *wq, struct cpu_info *ci)
|
||||
{
|
||||
struct workqueue_queue *q;
|
||||
|
||||
SLIST_FOREACH(q, &wq->wq_queue, q_list)
|
||||
if (q->q_ci == ci)
|
||||
return q;
|
||||
|
||||
return SLIST_FIRST(&wq->wq_queue);
|
||||
}
|
||||
|
||||
static void
|
||||
workqueue_runlist(struct workqueue *wq, struct workqhead *list)
|
||||
{
|
||||
|
|
@ -78,8 +91,12 @@ workqueue_runlist(struct workqueue *wq, struct workqhead *list)
|
|||
static void
|
||||
workqueue_run(struct workqueue *wq)
|
||||
{
|
||||
struct workqueue_queue *q = &wq->wq_queue;
|
||||
|
||||
struct workqueue_queue *q;
|
||||
|
||||
/* find the workqueue of this kthread */
|
||||
q = workqueue_queue_lookup(wq, curlwp->l_cpu);
|
||||
KASSERT(q != NULL);
|
||||
|
||||
for (;;) {
|
||||
struct workqhead tmp;
|
||||
|
||||
|
|
@ -128,20 +145,26 @@ workqueue_init(struct workqueue *wq, const char *name,
|
|||
wq->wq_name = name;
|
||||
wq->wq_func = callback_func;
|
||||
wq->wq_arg = callback_arg;
|
||||
SLIST_INIT(&wq->wq_queue);
|
||||
}
|
||||
|
||||
static int
|
||||
workqueue_initqueue(struct workqueue *wq, int ipl, int flags)
|
||||
workqueue_initqueue(struct workqueue *wq, int ipl,
|
||||
int flags, struct cpu_info *ci)
|
||||
{
|
||||
struct workqueue_queue *q = &wq->wq_queue;
|
||||
struct workqueue_queue *q;
|
||||
int error, ktf;
|
||||
|
||||
q = kmem_alloc(sizeof(struct workqueue_queue), KM_SLEEP);
|
||||
SLIST_INSERT_HEAD(&wq->wq_queue, q, q_list);
|
||||
q->q_ci = ci;
|
||||
|
||||
mutex_init(&q->q_mutex, MUTEX_DRIVER, ipl);
|
||||
cv_init(&q->q_cv, wq->wq_name);
|
||||
SIMPLEQ_INIT(&q->q_queue);
|
||||
ktf = ((flags & WQ_MPSAFE) != 0 ? KTHREAD_MPSAFE : 0);
|
||||
error = kthread_create(wq->wq_prio, ktf, NULL, workqueue_worker,
|
||||
wq, &q->q_worker, wq->wq_name);
|
||||
error = kthread_create(wq->wq_prio, ktf, ci, workqueue_worker,
|
||||
wq, &q->q_worker, "%s/%d", wq->wq_name, (int)ci->ci_cpuid);
|
||||
|
||||
return error;
|
||||
}
|
||||
|
|
@ -170,9 +193,8 @@ workqueue_exit(struct work *wk, void *arg)
|
|||
}
|
||||
|
||||
static void
|
||||
workqueue_finiqueue(struct workqueue *wq)
|
||||
workqueue_finiqueue(struct workqueue *wq, struct workqueue_queue *q)
|
||||
{
|
||||
struct workqueue_queue *q = &wq->wq_queue;
|
||||
struct workqueue_exitargs wqe;
|
||||
|
||||
wq->wq_func = workqueue_exit;
|
||||
|
|
@ -189,6 +211,7 @@ workqueue_finiqueue(struct workqueue *wq)
|
|||
mutex_exit(&q->q_mutex);
|
||||
mutex_destroy(&q->q_mutex);
|
||||
cv_destroy(&q->q_cv);
|
||||
kmem_free(q, sizeof(struct workqueue_queue));
|
||||
}
|
||||
|
||||
/* --- */
|
||||
|
|
@ -199,19 +222,31 @@ workqueue_create(struct workqueue **wqp, const char *name,
|
|||
pri_t prio, int ipl, int flags)
|
||||
{
|
||||
struct workqueue *wq;
|
||||
int error;
|
||||
int error = 0;
|
||||
|
||||
wq = kmem_alloc(sizeof(*wq), KM_SLEEP);
|
||||
if (wq == NULL) {
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
workqueue_init(wq, name, callback_func, callback_arg, prio, ipl);
|
||||
|
||||
error = workqueue_initqueue(wq, ipl, flags);
|
||||
if (error) {
|
||||
kmem_free(wq, sizeof(*wq));
|
||||
return error;
|
||||
if (flags & WQ_PERCPU) {
|
||||
struct cpu_info *ci;
|
||||
CPU_INFO_ITERATOR cii;
|
||||
|
||||
/* create the work-queue for each CPU */
|
||||
for (CPU_INFO_FOREACH(cii, ci)) {
|
||||
error = workqueue_initqueue(wq, ipl, flags, ci);
|
||||
if (error)
|
||||
break;
|
||||
}
|
||||
if (error)
|
||||
workqueue_destroy(wq);
|
||||
|
||||
} else {
|
||||
error = workqueue_initqueue(wq, ipl, flags, curcpu());
|
||||
if (error) {
|
||||
kmem_free(wq, sizeof(*wq));
|
||||
return error;
|
||||
}
|
||||
}
|
||||
|
||||
*wqp = wq;
|
||||
|
|
@ -221,15 +256,22 @@ workqueue_create(struct workqueue **wqp, const char *name,
|
|||
void
|
||||
workqueue_destroy(struct workqueue *wq)
|
||||
{
|
||||
struct workqueue_queue *q;
|
||||
|
||||
workqueue_finiqueue(wq);
|
||||
while ((q = SLIST_FIRST(&wq->wq_queue)) != NULL) {
|
||||
workqueue_finiqueue(wq, q);
|
||||
SLIST_REMOVE_HEAD(&wq->wq_queue, q_list);
|
||||
}
|
||||
kmem_free(wq, sizeof(*wq));
|
||||
}
|
||||
|
||||
void
|
||||
workqueue_enqueue(struct workqueue *wq, struct work *wk)
|
||||
workqueue_enqueue(struct workqueue *wq, struct work *wk, struct cpu_info *ci)
|
||||
{
|
||||
struct workqueue_queue *q = &wq->wq_queue;
|
||||
struct workqueue_queue *q;
|
||||
|
||||
q = workqueue_queue_lookup(wq, ci);
|
||||
KASSERT(q != NULL);
|
||||
|
||||
mutex_enter(&q->q_mutex);
|
||||
SIMPLEQ_INSERT_TAIL(&q->q_queue, wk, wk_entry);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: workqueue.h,v 1.5 2007/07/09 21:11:34 ad Exp $ */
|
||||
/* $NetBSD: workqueue.h,v 1.6 2007/07/12 20:39:56 rmind Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c)2002, 2005 YAMAMOTO Takashi,
|
||||
|
|
@ -46,11 +46,12 @@ struct work {
|
|||
struct workqueue;
|
||||
|
||||
#define WQ_MPSAFE 0x01
|
||||
#define WQ_PERCPU 0x02
|
||||
|
||||
int workqueue_create(struct workqueue **, const char *,
|
||||
void (*)(struct work *, void *), void *, pri_t, int, int);
|
||||
void workqueue_destroy(struct workqueue *);
|
||||
|
||||
void workqueue_enqueue(struct workqueue *, struct work *);
|
||||
void workqueue_enqueue(struct workqueue *, struct work *, struct cpu_info *);
|
||||
|
||||
#endif /* _SYS_WORKQUEUE_H_ */
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: lfs_segment.c,v 1.201 2007/06/30 09:37:53 pooka Exp $ */
|
||||
/* $NetBSD: lfs_segment.c,v 1.202 2007/07/12 20:39:57 rmind Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc.
|
||||
|
|
@ -67,7 +67,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.201 2007/06/30 09:37:53 pooka Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.202 2007/07/12 20:39:57 rmind Exp $");
|
||||
|
||||
#ifdef DEBUG
|
||||
# define vndebug(vp, str) do { \
|
||||
|
|
@ -2591,7 +2591,7 @@ lfs_generic_callback(struct buf *bp, void (*aiodone)(struct buf *))
|
|||
/* reset b_iodone for when this is a single-buf i/o. */
|
||||
bp->b_iodone = aiodone;
|
||||
|
||||
workqueue_enqueue(uvm.aiodone_queue, &bp->b_work);
|
||||
workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_pager.c,v 1.82 2007/07/09 21:11:37 ad Exp $ */
|
||||
/* $NetBSD: uvm_pager.c,v 1.83 2007/07/12 20:39:57 rmind Exp $ */
|
||||
|
||||
/*
|
||||
*
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.82 2007/07/09 21:11:37 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.83 2007/07/12 20:39:57 rmind Exp $");
|
||||
|
||||
#include "opt_uvmhist.h"
|
||||
#include "opt_readahead.h"
|
||||
|
|
@ -275,7 +275,7 @@ uvm_aio_biodone(struct buf *bp)
|
|||
/* reset b_iodone for when this is a single-buf i/o. */
|
||||
bp->b_iodone = uvm_aio_aiodone;
|
||||
|
||||
workqueue_enqueue(uvm.aiodone_queue, &bp->b_work);
|
||||
workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue