Aren
/
NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add uvm_kick_scheduler() (MP safe) to replace wakeup(&proc0).

This commit is contained in:
ad 2007-02-15 20:21:13 +00:00
parent cd12688f17
commit d91014721f
8 changed files with 139 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
sys/kern/kern_sleepq.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: kern_sleepq.c,v 1.3 2007/02/10 14:02:01 yamt Exp $ */ /* $NetBSD: kern_sleepq.c,v 1.4 2007/02/15 20:21:13 ad Exp $ */
/*- /*-
* Copyright (c) 2006, 2007 The NetBSD Foundation, Inc. * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
@ -42,7 +42,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.3 2007/02/10 14:02:01 yamt Exp $"); __KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.4 2007/02/15 20:21:13 ad Exp $");
#include "opt_multiprocessor.h" #include "opt_multiprocessor.h"
#include "opt_lockdebug.h" #include "opt_lockdebug.h"
@ -57,11 +57,12 @@ __KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.3 2007/02/10 14:02:01 yamt Exp $")
#include <sys/sched.h> #include <sys/sched.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/sleepq.h> #include <sys/sleepq.h>
#ifdef KTRACE #ifdef KTRACE
#include <sys/ktrace.h> #include <sys/ktrace.h>
#endif #endif
#include <uvm/uvm_extern.h>
int sleepq_sigtoerror(struct lwp *, int); int sleepq_sigtoerror(struct lwp *, int);
void updatepri(struct lwp *); void updatepri(struct lwp *);
@ -117,7 +118,7 @@ sleepq_remove(sleepq_t *sq, struct lwp *l)
{ {
struct cpu_info *ci; struct cpu_info *ci;
LOCK_ASSERT(lwp_locked(l, sq->sq_mutex)); KASSERT(lwp_locked(l, sq->sq_mutex));
KASSERT(sq->sq_waiters > 0); KASSERT(sq->sq_waiters > 0);
sq->sq_waiters--; sq->sq_waiters--;
@ -224,15 +225,10 @@ sleepq_block(sleepq_t *sq, int pri, wchan_t wchan, const char *wmesg, int timo,
{ {
struct lwp *l = curlwp; struct lwp *l = curlwp;
LOCK_ASSERT(mutex_owned(sq->sq_mutex)); KASSERT(mutex_owned(sq->sq_mutex));
KASSERT(l->l_stat == LSONPROC); KASSERT(l->l_stat == LSONPROC);
KASSERT(l->l_wchan == NULL && l->l_sleepq == NULL); KASSERT(l->l_wchan == NULL && l->l_sleepq == NULL);
#ifdef KTRACE
if (KTRPOINT(l->l_proc, KTR_CSW))
ktrcsw(l, 1, 0);
#endif
l->l_syncobj = sobj; l->l_syncobj = sobj;
l->l_wchan = wchan; l->l_wchan = wchan;
l->l_sleepq = sq; l->l_sleepq = sq;
@ -246,6 +242,11 @@ sleepq_block(sleepq_t *sq, int pri, wchan_t wchan, const char *wmesg, int timo,
sq->sq_waiters++; sq->sq_waiters++;
sleepq_insert(sq, l, pri, sobj); sleepq_insert(sq, l, pri, sobj);
#ifdef KTRACE
if (KTRPOINT(l->l_proc, KTR_CSW))
ktrcsw(l, 1, 0);
#endif
/* /*
* If sleeping interruptably, check for pending signals, exits or * If sleeping interruptably, check for pending signals, exits or
* core dump events. * core dump events.
@ -350,7 +351,7 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
struct lwp *l, *next; struct lwp *l, *next;
int swapin = 0; int swapin = 0;
LOCK_ASSERT(mutex_owned(sq->sq_mutex)); KASSERT(mutex_owned(sq->sq_mutex));
for (l = TAILQ_FIRST(&sq->sq_queue); l != NULL; l = next) { for (l = TAILQ_FIRST(&sq->sq_queue); l != NULL; l = next) {
KASSERT(l->l_sleepq == sq); KASSERT(l->l_sleepq == sq);
@ -362,7 +363,6 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
break; break;
} }
LOCK_ASSERT(mutex_owned(sq->sq_mutex));
sleepq_unlock(sq); sleepq_unlock(sq);
/* /*
@ -370,7 +370,7 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
* then kick the swapper into action. * then kick the swapper into action.
*/ */
if (swapin) if (swapin)
wakeup(&proc0); uvm_kick_scheduler();
} }
/* /*
@ -386,7 +386,7 @@ sleepq_unsleep(struct lwp *l)
sleepq_t *sq = l->l_sleepq; sleepq_t *sq = l->l_sleepq;
int swapin; int swapin;
LOCK_ASSERT(lwp_locked(l, NULL)); KASSERT(lwp_locked(l, NULL));
KASSERT(l->l_wchan != NULL); KASSERT(l->l_wchan != NULL);
KASSERT(l->l_mutex == sq->sq_mutex); KASSERT(l->l_mutex == sq->sq_mutex);
@ -394,7 +394,7 @@ sleepq_unsleep(struct lwp *l)
sleepq_unlock(sq); sleepq_unlock(sq);
if (swapin) if (swapin)
wakeup(&proc0); uvm_kick_scheduler();
} }
/* /*
@ -434,7 +434,7 @@ sleepq_sigtoerror(struct lwp *l, int sig)
struct proc *p = l->l_proc; struct proc *p = l->l_proc;
int error; int error;
LOCK_ASSERT(mutex_owned(&p->p_smutex)); KASSERT(mutex_owned(&p->p_smutex));
/* /*
* If this sleep was canceled, don't let the syscall restart. * If this sleep was canceled, don't let the syscall restart.

6
sys/kern/kern_synch.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: kern_synch.c,v 1.176 2007/02/10 14:02:01 yamt Exp $ */ /* $NetBSD: kern_synch.c,v 1.177 2007/02/15 20:21:13 ad Exp $ */
/*- /*-
* Copyright (c) 1999, 2000, 2004, 2006, 2007 The NetBSD Foundation, Inc. * Copyright (c) 1999, 2000, 2004, 2006, 2007 The NetBSD Foundation, Inc.
@ -74,7 +74,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.176 2007/02/10 14:02:01 yamt Exp $"); __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.177 2007/02/15 20:21:13 ad Exp $");
#include "opt_ddb.h" #include "opt_ddb.h"
#include "opt_kstack.h" #include "opt_kstack.h"
@ -862,7 +862,7 @@ setrunnable(struct lwp *l)
lwp_unlock(l); lwp_unlock(l);
} else { } else {
lwp_unlock(l); lwp_unlock(l);
wakeup(&proc0); uvm_kick_scheduler();
} }
} }

8
sys/kern/kern_turnstile.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: kern_turnstile.c,v 1.2 2007/02/09 21:55:31 ad Exp $ */ /* $NetBSD: kern_turnstile.c,v 1.3 2007/02/15 20:21:13 ad Exp $ */
/*- /*-
* Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc. * Copyright (c) 2002, 2006, 2007 The NetBSD Foundation, Inc.
@ -68,7 +68,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.2 2007/02/09 21:55:31 ad Exp $"); __KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.3 2007/02/15 20:21:13 ad Exp $");
#include "opt_lockdebug.h" #include "opt_lockdebug.h"
#include "opt_multiprocessor.h" #include "opt_multiprocessor.h"
@ -82,6 +82,8 @@ __KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.2 2007/02/09 21:55:31 ad Exp $"
#include <sys/sleepq.h> #include <sys/sleepq.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <uvm/uvm_extern.h>
#define TS_HASH_SIZE 64 #define TS_HASH_SIZE 64
#define TS_HASH_MASK (TS_HASH_SIZE - 1) #define TS_HASH_MASK (TS_HASH_SIZE - 1)
#define TS_HASH(obj) (((uintptr_t)(obj) >> 3) & TS_HASH_MASK) #define TS_HASH(obj) (((uintptr_t)(obj) >> 3) & TS_HASH_MASK)
@ -324,7 +326,7 @@ turnstile_wakeup(turnstile_t *ts, int q, int count, struct lwp *nl)
* then kick the swapper into action. * then kick the swapper into action.
*/ */
if (swapin) if (swapin)
wakeup(&proc0); uvm_kick_scheduler();
} }
/* /*

8
sys/kern/sys_lwp.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: sys_lwp.c,v 1.2 2007/02/09 21:55:31 ad Exp $ */ /* $NetBSD: sys_lwp.c,v 1.3 2007/02/15 20:21:13 ad Exp $ */
/*- /*-
* Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc. * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
@ -42,7 +42,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.2 2007/02/09 21:55:31 ad Exp $"); __KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.3 2007/02/15 20:21:13 ad Exp $");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
@ -568,7 +568,7 @@ sys__lwp_unpark(struct lwp *l, void *v, register_t *retval)
swapin = sleepq_remove(sq, t); swapin = sleepq_remove(sq, t);
sleepq_unlock(sq); sleepq_unlock(sq);
if (swapin) if (swapin)
wakeup(&proc0); uvm_kick_scheduler();
LWP_COUNT(lwp_ev_park_targ, 1); LWP_COUNT(lwp_ev_park_targ, 1);
return 0; return 0;
} }
@ -693,7 +693,7 @@ sys__lwp_unpark_all(struct lwp *l, void *v, register_t *retval)
KERNEL_UNLOCK_ONE(l); /* XXXSMP */ KERNEL_UNLOCK_ONE(l); /* XXXSMP */
} }
if (swapin) if (swapin)
wakeup(&proc0); uvm_kick_scheduler();
LWP_COUNT(lwp_ev_park_bcast, unparked); LWP_COUNT(lwp_ev_park_bcast, unparked);
LWP_COUNT(lwp_ev_park_miss, (ntargets - unparked)); LWP_COUNT(lwp_ev_park_miss, (ntargets - unparked));
/* XXXAD return unparked; */ /* XXXAD return unparked; */

6
sys/uvm/uvm.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm.h,v 1.45 2006/12/21 15:55:26 yamt Exp $ */ /* $NetBSD: uvm.h,v 1.46 2007/02/15 20:21:14 ad Exp $ */
/* /*
* *
@ -111,9 +111,13 @@ struct uvm {
/* swap-related items */ /* swap-related items */
struct simplelock swap_data_lock; struct simplelock swap_data_lock;
kcondvar_t scheduler_cv;
kmutex_t scheduler_mutex;
boolean_t scheduler_kicked;
/* kernel object: to support anonymous pageable kernel memory */ /* kernel object: to support anonymous pageable kernel memory */
struct uvm_object *kernel_object; struct uvm_object *kernel_object;
}; };
#endif /* _KERNEL */ #endif /* _KERNEL */

3
sys/uvm/uvm_extern.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_extern.h,v 1.124 2006/12/21 15:55:26 yamt Exp $ */ /* $NetBSD: uvm_extern.h,v 1.125 2007/02/15 20:21:13 ad Exp $ */
/* /*
* *
@ -578,6 +578,7 @@ void uvm_lwp_exit(struct lwp *);
void uvm_init_limits(struct proc *); void uvm_init_limits(struct proc *);
boolean_t uvm_kernacc(caddr_t, size_t, int); boolean_t uvm_kernacc(caddr_t, size_t, int);
__dead void uvm_scheduler(void) __attribute__((noreturn)); __dead void uvm_scheduler(void) __attribute__((noreturn));
void uvm_kick_scheduler(void);
void uvm_swapin(struct lwp *); void uvm_swapin(struct lwp *);
boolean_t uvm_uarea_alloc(vaddr_t *); boolean_t uvm_uarea_alloc(vaddr_t *);
void uvm_uarea_drain(boolean_t); void uvm_uarea_drain(boolean_t);

156
sys/uvm/uvm_glue.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_glue.c,v 1.98 2007/02/09 21:55:43 ad Exp $ */ /* $NetBSD: uvm_glue.c,v 1.99 2007/02/15 20:21:13 ad Exp $ */
/* /*
* Copyright (c) 1997 Charles D. Cranor and Washington University. * Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -67,7 +67,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.98 2007/02/09 21:55:43 ad Exp $"); __KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.99 2007/02/15 20:21:13 ad Exp $");
#include "opt_coredump.h" #include "opt_coredump.h"
#include "opt_kgdb.h" #include "opt_kgdb.h"
@ -450,6 +450,22 @@ uvm_swapin(struct lwp *l)
++uvmexp.swapins; ++uvmexp.swapins;
} }
/*
* uvm_kick_scheduler: kick the scheduler into action if not running.
*
* - called when swapped out processes have been awoken.
*/
void
uvm_kick_scheduler(void)
{
mutex_enter(&uvm.scheduler_mutex);
uvm.scheduler_kicked = TRUE;
cv_signal(&uvm.scheduler_cv);
mutex_exit(&uvm.scheduler_mutex);
}
/* /*
* uvm_scheduler: process zero main loop * uvm_scheduler: process zero main loop
* *
@ -465,75 +481,87 @@ uvm_scheduler(void)
int pri; int pri;
int ppri; int ppri;
loop: l = curlwp;
#ifdef DEBUG lwp_lock(l);
while (!enableswap) lwp_changepri(l, PVM);
tsleep(&proc0, PVM, "noswap", 0); lwp_unlock(l);
#endif
ll = NULL; /* process to choose */
ppri = INT_MIN; /* its priority */
mutex_enter(&proclist_mutex); for (;;) {
LIST_FOREACH(l, &alllwp, l_list) { #ifdef DEBUG
/* is it a runnable swapped out process? */ mutex_enter(&uvm.scheduler_mutex);
if (l->l_stat == LSRUN && (l->l_flag & L_INMEM) == 0) { while (!enableswap)
pri = l->l_swtime + l->l_slptime - cv_wait(&uvm.scheduler_cv, &uvm.scheduler_mutex);
(l->l_proc->p_nice - NZERO) * 8; mutex_exit(&uvm.scheduler_mutex);
if (pri > ppri) { /* higher priority? remember it. */ #endif
ll = l; ll = NULL; /* process to choose */
ppri = pri; ppri = INT_MIN; /* its priority */
mutex_enter(&proclist_mutex);
LIST_FOREACH(l, &alllwp, l_list) {
/* is it a runnable swapped out process? */
if (l->l_stat == LSRUN && (l->l_flag & L_INMEM) == 0) {
pri = l->l_swtime + l->l_slptime -
(l->l_proc->p_nice - NZERO) * 8;
if (pri > ppri) { /* higher priority? */
ll = l;
ppri = pri;
}
} }
} }
} mutex_exit(&proclist_mutex);
/* #ifdef DEBUG
* XXXSMP: possible unlock/sleep race between here and the if (swapdebug & SDB_FOLLOW)
* "scheduler" tsleep below.. printf("scheduler: running, procp %p pri %d\n", ll,
*/ ppri);
mutex_exit(&proclist_mutex); #endif
/*
* Nothing to do, back to sleep
*/
if ((l = ll) == NULL) {
mutex_enter(&uvm.scheduler_mutex);
if (uvm.scheduler_kicked == FALSE)
cv_wait(&uvm.scheduler_cv,
&uvm.scheduler_mutex);
uvm.scheduler_kicked = FALSE;
mutex_exit(&uvm.scheduler_mutex);
continue;
}
/*
* we have found swapped out process which we would like
* to bring back in.
*
* XXX: this part is really bogus cuz we could deadlock
* on memory despite our feeble check
*/
if (uvmexp.free > atop(USPACE)) {
#ifdef DEBUG #ifdef DEBUG
if (swapdebug & SDB_FOLLOW) if (swapdebug & SDB_SWAPIN)
printf("scheduler: running, procp %p pri %d\n", ll, ppri); printf("swapin: pid %d(%s)@%p, pri %d "
"free %d\n", l->l_proc->p_pid,
l->l_proc->p_comm, l->l_addr, ppri,
uvmexp.free);
#endif #endif
/* uvm_swapin(l);
* Nothing to do, back to sleep } else {
*/ /*
if ((l = ll) == NULL) { * not enough memory, jab the pageout daemon and
tsleep(&proc0, PVM, "scheduler", 0); * wait til the coast is clear
goto loop; */
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: no room for pid %d(%s),"
" free %d\n", l->l_proc->p_pid,
l->l_proc->p_comm, uvmexp.free);
#endif
uvm_wait("schedpwait");
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: room again, free %d\n",
uvmexp.free);
#endif
}
} }
/*
* we have found swapped out process which we would like to bring
* back in.
*
* XXX: this part is really bogus cuz we could deadlock on memory
* despite our feeble check
*/
if (uvmexp.free > atop(USPACE)) {
#ifdef DEBUG
if (swapdebug & SDB_SWAPIN)
printf("swapin: pid %d(%s)@%p, pri %d free %d\n",
l->l_proc->p_pid, l->l_proc->p_comm, l->l_addr, ppri, uvmexp.free);
#endif
uvm_swapin(l);
goto loop;
}
/*
* not enough memory, jab the pageout daemon and wait til the coast
* is clear
*/
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: no room for pid %d(%s), free %d\n",
l->l_proc->p_pid, l->l_proc->p_comm, uvmexp.free);
#endif
uvm_wait("schedpwait");
#ifdef DEBUG
if (swapdebug & SDB_FOLLOW)
printf("scheduler: room again, free %d\n", uvmexp.free);
#endif
goto loop;
} }
/* /*

20
sys/uvm/uvm_swap.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_swap.c,v 1.116 2007/02/09 21:55:43 ad Exp $ */ /* $NetBSD: uvm_swap.c,v 1.117 2007/02/15 20:21:13 ad Exp $ */
/* /*
* Copyright (c) 1995, 1996, 1997 Matthew R. Green * Copyright (c) 1995, 1996, 1997 Matthew R. Green
@ -32,7 +32,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.116 2007/02/09 21:55:43 ad Exp $"); __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.117 2007/02/15 20:21:13 ad Exp $");
#include "fs_nfs.h" #include "fs_nfs.h"
#include "opt_uvmhist.h" #include "opt_uvmhist.h"
@ -215,7 +215,7 @@ LIST_HEAD(swap_priority, swappri);
static struct swap_priority swap_priority; static struct swap_priority swap_priority;
/* locks */ /* locks */
static struct lock swap_syscall_lock; static krwlock_t swap_syscall_lock;
/* /*
* prototypes * prototypes
@ -258,9 +258,13 @@ uvm_swap_init(void)
LIST_INIT(&swap_priority); LIST_INIT(&swap_priority);
uvmexp.nswapdev = 0; uvmexp.nswapdev = 0;
lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0); rw_init(&swap_syscall_lock);
cv_init(&uvm.scheduler_cv, "schedule");
simple_lock_init(&uvm.swap_data_lock); simple_lock_init(&uvm.swap_data_lock);
/* XXXSMP should be at IPL_VM, but for audio interrupt handlers. */
mutex_init(&uvm.scheduler_mutex, MUTEX_SPIN, IPL_SCHED);
if (bdevvp(swapdev, &swapdev_vp)) if (bdevvp(swapdev, &swapdev_vp))
panic("uvm_swap_init: can't get vnode for swap device"); panic("uvm_swap_init: can't get vnode for swap device");
@ -448,7 +452,7 @@ sys_swapctl(struct lwp *l, void *v, register_t *retval)
/* /*
* ensure serialized syscall access by grabbing the swap_syscall_lock * ensure serialized syscall access by grabbing the swap_syscall_lock
*/ */
lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); rw_enter(&swap_syscall_lock, RW_WRITER);
userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK); userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK);
/* /*
@ -677,7 +681,7 @@ sys_swapctl(struct lwp *l, void *v, register_t *retval)
out: out:
free(userpath, M_TEMP); free(userpath, M_TEMP);
lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); rw_exit(&swap_syscall_lock);
UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0);
return (error); return (error);
@ -696,9 +700,9 @@ void
uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval)
{ {
lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); rw_enter(&swap_syscall_lock, RW_READER);
uvm_swap_stats_locked(cmd, sep, sec, retval); uvm_swap_stats_locked(cmd, sep, sec, retval);
lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); rw_exit(&swap_syscall_lock);
} }
static void static void