Add uvm_kick_scheduler() (MP safe) to replace wakeup(&proc0).

This commit is contained in:
ad 2007-02-15 20:21:13 +00:00
parent cd12688f17
commit d91014721f
8 changed files with 139 additions and 100 deletions

View File

@ -1,4 +1,4 @@
/* $NetBSD: kern_sleepq.c,v 1.3 2007/02/10 14:02:01 yamt Exp $ */
/* $NetBSD: kern_sleepq.c,v 1.4 2007/02/15 20:21:13 ad Exp $ */
/*-
* Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
@ -42,7 +42,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.3 2007/02/10 14:02:01 yamt Exp $");
__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.4 2007/02/15 20:21:13 ad Exp $");
#include "opt_multiprocessor.h"
#include "opt_lockdebug.h"
@ -57,11 +57,12 @@ __KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.3 2007/02/10 14:02:01 yamt Exp $")
#include <sys/sched.h>
#include <sys/systm.h>
#include <sys/sleepq.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#include <uvm/uvm_extern.h>
int sleepq_sigtoerror(struct lwp *, int);
void updatepri(struct lwp *);
@ -117,7 +118,7 @@ sleepq_remove(sleepq_t *sq, struct lwp *l)
{
struct cpu_info *ci;
LOCK_ASSERT(lwp_locked(l, sq->sq_mutex));
KASSERT(lwp_locked(l, sq->sq_mutex));
KASSERT(sq->sq_waiters > 0);
sq->sq_waiters--;
@ -224,15 +225,10 @@ sleepq_block(sleepq_t *sq, int pri, wchan_t wchan, const char *wmesg, int timo,
{
struct lwp *l = curlwp;
LOCK_ASSERT(mutex_owned(sq->sq_mutex));
KASSERT(mutex_owned(sq->sq_mutex));
KASSERT(l->l_stat == LSONPROC);
KASSERT(l->l_wchan == NULL && l->l_sleepq == NULL);
#ifdef KTRACE
if (KTRPOINT(l->l_proc, KTR_CSW))
ktrcsw(l, 1, 0);
#endif
l->l_syncobj = sobj;
l->l_wchan = wchan;
l->l_sleepq = sq;
@ -246,6 +242,11 @@ sleepq_block(sleepq_t *sq, int pri, wchan_t wchan, const char *wmesg, int timo,
sq->sq_waiters++;
sleepq_insert(sq, l, pri, sobj);
#ifdef KTRACE
if (KTRPOINT(l->l_proc, KTR_CSW))
ktrcsw(l, 1, 0);
#endif
/*
* If sleeping interruptably, check for pending signals, exits or
* core dump events.
@ -350,7 +351,7 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
struct lwp *l, *next;
int swapin = 0;
LOCK_ASSERT(mutex_owned(sq->sq_mutex));
KASSERT(mutex_owned(sq->sq_mutex));
for (l = TAILQ_FIRST(&sq->sq_queue); l != NULL; l = next) {
KASSERT(l->l_sleepq == sq);
@ -362,7 +363,6 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
break;
}
LOCK_ASSERT(mutex_owned(sq->sq_mutex));
sleepq_unlock(sq);
/*
@ -370,7 +370,7 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
* then kick the swapper into action.
*/
if (swapin)
wakeup(&proc0);
uvm_kick_scheduler();
}
/*
@ -386,7 +386,7 @@ sleepq_unsleep(struct lwp *l)
sleepq_t *sq = l->l_sleepq;
int swapin;
LOCK_ASSERT(lwp_locked(l, NULL));
KASSERT(lwp_locked(l, NULL));
KASSERT(l->l_wchan != NULL);
KASSERT(l->l_mutex == sq->sq_mutex);
@ -394,7 +394,7 @@ sleepq_unsleep(struct lwp *l)
sleepq_unlock(sq);
if (swapin)
wakeup(&proc0);
uvm_kick_scheduler();
}
/*
@ -434,7 +434,7 @@ sleepq_sigtoerror(struct lwp *l, int sig)
struct proc *p = l->l_proc;
int error;
LOCK_ASSERT(mutex_owned(&p->p_smutex));
KASSERT(mutex_owned(&p->p_smutex));
/*
* If this sleep was canceled, don't let the syscall restart.

View File

@ -1,4 +1,4 @@
/* $NetBSD: kern_synch.c,v 1.176 2007/02/10 14:02:01 yamt Exp $ */
/* $NetBSD: kern_synch.c,v 1.177 2007/02/15 20:21:13 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2004, 2006, 2007 The NetBSD Foundation, Inc.
@ -74,7 +74,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.176 2007/02/10 14:02:01 yamt Exp $");
__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.177 2007/02/15 20:21:13 ad Exp $");
#include "opt_ddb.h"
#include "opt_kstack.h"
@ -862,7 +862,7 @@ setrunnable(struct lwp *l)
lwp_unlock(l);
} else {
lwp_unlock(l);
wakeup(&proc0);
uvm_kick_scheduler();
}
}

View File

@ -1,4 +1,4 @@
/* $NetBSD: kern_turnstile.c,v 1.2 2007/02/09 21:55:31 ad Exp $ */
/* $NetBSD: kern_turnstile.c,v 1.3 2007/02/15 20:21:13 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
@ -68,7 +68,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.2 2007/02/09 21:55:31 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.3 2007/02/15 20:21:13 ad Exp $");
#include "opt_lockdebug.h"
#include "opt_multiprocessor.h"
@ -82,6 +82,8 @@ __KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.2 2007/02/09 21:55:31 ad Exp $"
#include <sys/sleepq.h>
#include <sys/systm.h>
#include <uvm/uvm_extern.h>
#define TS_HASH_SIZE 64
#define TS_HASH_MASK (TS_HASH_SIZE - 1)
#define TS_HASH(obj) (((uintptr_t)(obj) >> 3) & TS_HASH_MASK)
@ -324,7 +326,7 @@ turnstile_wakeup(turnstile_t *ts, int q, int count, struct lwp *nl)
* then kick the swapper into action.
*/
if (swapin)
wakeup(&proc0);
uvm_kick_scheduler();
}
/*

View File

@ -1,4 +1,4 @@
/* $NetBSD: sys_lwp.c,v 1.2 2007/02/09 21:55:31 ad Exp $ */
/* $NetBSD: sys_lwp.c,v 1.3 2007/02/15 20:21:13 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
@ -42,7 +42,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.2 2007/02/09 21:55:31 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.3 2007/02/15 20:21:13 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -568,7 +568,7 @@ sys__lwp_unpark(struct lwp *l, void *v, register_t *retval)
swapin = sleepq_remove(sq, t);
sleepq_unlock(sq);
if (swapin)
wakeup(&proc0);
uvm_kick_scheduler();
LWP_COUNT(lwp_ev_park_targ, 1);
return 0;
}
@ -693,7 +693,7 @@ sys__lwp_unpark_all(struct lwp *l, void *v, register_t *retval)
KERNEL_UNLOCK_ONE(l); /* XXXSMP */
}
if (swapin)
wakeup(&proc0);
uvm_kick_scheduler();
LWP_COUNT(lwp_ev_park_bcast, unparked);
LWP_COUNT(lwp_ev_park_miss, (ntargets - unparked));
/* XXXAD return unparked; */

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm.h,v 1.45 2006/12/21 15:55:26 yamt Exp $ */
/* $NetBSD: uvm.h,v 1.46 2007/02/15 20:21:14 ad Exp $ */
/*
*
@ -111,9 +111,13 @@ struct uvm {
/* swap-related items */
struct simplelock swap_data_lock;
kcondvar_t scheduler_cv;
kmutex_t scheduler_mutex;
boolean_t scheduler_kicked;
/* kernel object: to support anonymous pageable kernel memory */
struct uvm_object *kernel_object;
};
#endif /* _KERNEL */

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_extern.h,v 1.124 2006/12/21 15:55:26 yamt Exp $ */
/* $NetBSD: uvm_extern.h,v 1.125 2007/02/15 20:21:13 ad Exp $ */
/*
*
@ -578,6 +578,7 @@ void uvm_lwp_exit(struct lwp *);
void uvm_init_limits(struct proc *);
boolean_t uvm_kernacc(caddr_t, size_t, int);
__dead void uvm_scheduler(void) __attribute__((noreturn));
void uvm_kick_scheduler(void);
void uvm_swapin(struct lwp *);
boolean_t uvm_uarea_alloc(vaddr_t *);
void uvm_uarea_drain(boolean_t);

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_glue.c,v 1.98 2007/02/09 21:55:43 ad Exp $ */
/* $NetBSD: uvm_glue.c,v 1.99 2007/02/15 20:21:13 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -67,7 +67,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.98 2007/02/09 21:55:43 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.99 2007/02/15 20:21:13 ad Exp $");
#include "opt_coredump.h"
#include "opt_kgdb.h"
@ -450,6 +450,22 @@ uvm_swapin(struct lwp *l)
++uvmexp.swapins;
}
/*
* uvm_kick_scheduler: kick the scheduler into action if not running.
*
* - called when swapped out processes have been awoken.
*/
void
uvm_kick_scheduler(void)
{
mutex_enter(&uvm.scheduler_mutex);
uvm.scheduler_kicked = TRUE;
cv_signal(&uvm.scheduler_cv);
mutex_exit(&uvm.scheduler_mutex);
}
/*
* uvm_scheduler: process zero main loop
*
@ -465,75 +481,87 @@ uvm_scheduler(void)
int pri;
int ppri;
loop:
#ifdef DEBUG
while (!enableswap)
tsleep(&proc0, PVM, "noswap", 0);
#endif
ll = NULL; /* process to choose */
ppri = INT_MIN; /* its priority */
l = curlwp;
lwp_lock(l);
lwp_changepri(l, PVM);
lwp_unlock(l);
mutex_enter(&proclist_mutex);
LIST_FOREACH(l, &alllwp, l_list) {
/* is it a runnable swapped out process? */
if (l->l_stat == LSRUN && (l->l_flag & L_INMEM) == 0) {
pri = l->l_swtime + l->l_slptime -
(l->l_proc->p_nice - NZERO) * 8;
if (pri > ppri) { /* higher priority? remember it. */
ll = l;
ppri = pri;
for (;;) {
#ifdef DEBUG
mutex_enter(&uvm.scheduler_mutex);
while (!enableswap)
cv_wait(&uvm.scheduler_cv, &uvm.scheduler_mutex);
mutex_exit(&uvm.scheduler_mutex);
#endif
ll = NULL; /* process to choose */
ppri = INT_MIN; /* its priority */
mutex_enter(&proclist_mutex);
LIST_FOREACH(l, &alllwp, l_list) {
/* is it a runnable swapped out process? */
if (l->l_stat == LSRUN && (l->l_flag & L_INMEM) == 0) {
pri = l->l_swtime + l->l_slptime -
(l->l_proc->p_nice - NZERO) * 8;
if (pri > ppri) { /* higher priority? */
ll = l;
ppri = pri;
}
}
}
}
/*
* XXXSMP: possible unlock/sleep race between here and the
* "scheduler" tsleep below..
*/
mutex_exit(&proclist_mutex);
mutex_exit(&proclist_mutex);
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: running, procp %p pri %d\n", ll,
ppri);
#endif
/*
* Nothing to do, back to sleep
*/
if ((l = ll) == NULL) {
mutex_enter(&uvm.scheduler_mutex);
if (uvm.scheduler_kicked == FALSE)
cv_wait(&uvm.scheduler_cv,
&uvm.scheduler_mutex);
uvm.scheduler_kicked = FALSE;
mutex_exit(&uvm.scheduler_mutex);
continue;
}
/*
* we have found swapped out process which we would like
* to bring back in.
*
* XXX: this part is really bogus cuz we could deadlock
* on memory despite our feeble check
*/
if (uvmexp.free > atop(USPACE)) {
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: running, procp %p pri %d\n", ll, ppri);
if (swapdebug & SDB_SWAPIN)
printf("swapin: pid %d(%s)@%p, pri %d "
"free %d\n", l->l_proc->p_pid,
l->l_proc->p_comm, l->l_addr, ppri,
uvmexp.free);
#endif
/*
* Nothing to do, back to sleep
*/
if ((l = ll) == NULL) {
tsleep(&proc0, PVM, "scheduler", 0);
goto loop;
uvm_swapin(l);
} else {
/*
* not enough memory, jab the pageout daemon and
* wait til the coast is clear
*/
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: no room for pid %d(%s),"
" free %d\n", l->l_proc->p_pid,
l->l_proc->p_comm, uvmexp.free);
#endif
uvm_wait("schedpwait");
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: room again, free %d\n",
uvmexp.free);
#endif
}
}
/*
* we have found swapped out process which we would like to bring
* back in.
*
* XXX: this part is really bogus cuz we could deadlock on memory
* despite our feeble check
*/
if (uvmexp.free > atop(USPACE)) {
#ifdef DEBUG
if (swapdebug & SDB_SWAPIN)
printf("swapin: pid %d(%s)@%p, pri %d free %d\n",
l->l_proc->p_pid, l->l_proc->p_comm, l->l_addr, ppri, uvmexp.free);
#endif
uvm_swapin(l);
goto loop;
}
/*
* not enough memory, jab the pageout daemon and wait til the coast
* is clear
*/
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: no room for pid %d(%s), free %d\n",
l->l_proc->p_pid, l->l_proc->p_comm, uvmexp.free);
#endif
uvm_wait("schedpwait");
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: room again, free %d\n", uvmexp.free);
#endif
goto loop;
}
/*

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_swap.c,v 1.116 2007/02/09 21:55:43 ad Exp $ */
/* $NetBSD: uvm_swap.c,v 1.117 2007/02/15 20:21:13 ad Exp $ */
/*
* Copyright (c) 1995, 1996, 1997 Matthew R. Green
@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.116 2007/02/09 21:55:43 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.117 2007/02/15 20:21:13 ad Exp $");
#include "fs_nfs.h"
#include "opt_uvmhist.h"
@ -215,7 +215,7 @@ LIST_HEAD(swap_priority, swappri);
static struct swap_priority swap_priority;
/* locks */
static struct lock swap_syscall_lock;
static krwlock_t swap_syscall_lock;
/*
* prototypes
@ -258,9 +258,13 @@ uvm_swap_init(void)
LIST_INIT(&swap_priority);
uvmexp.nswapdev = 0;
lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0);
rw_init(&swap_syscall_lock);
cv_init(&uvm.scheduler_cv, "schedule");
simple_lock_init(&uvm.swap_data_lock);
/* XXXSMP should be at IPL_VM, but for audio interrupt handlers. */
mutex_init(&uvm.scheduler_mutex, MUTEX_SPIN, IPL_SCHED);
if (bdevvp(swapdev, &swapdev_vp))
panic("uvm_swap_init: can't get vnode for swap device");
@ -448,7 +452,7 @@ sys_swapctl(struct lwp *l, void *v, register_t *retval)
/*
* ensure serialized syscall access by grabbing the swap_syscall_lock
*/
lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL);
rw_enter(&swap_syscall_lock, RW_WRITER);
userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK);
/*
@ -677,7 +681,7 @@ sys_swapctl(struct lwp *l, void *v, register_t *retval)
out:
free(userpath, M_TEMP);
lockmgr(&swap_syscall_lock, LK_RELEASE, NULL);
rw_exit(&swap_syscall_lock);
UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0);
return (error);
@ -696,9 +700,9 @@ void
uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval)
{
lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL);
rw_enter(&swap_syscall_lock, RW_READER);
uvm_swap_stats_locked(cmd, sep, sec, retval);
lockmgr(&swap_syscall_lock, LK_RELEASE, NULL);
rw_exit(&swap_syscall_lock);
}
static void