- vm_page: put listq, pageq into a union alongside a LIST_ENTRY, so we can
use both types of list. - Make page coloring and idle zero state per-CPU. - Maintain per-CPU page freelists. When freeing, put pages onto the local CPU's lists and the global lists. When allocating, prefer to take pages from the local CPU. If none are available take from the global list as done now. Proposed on tech-kern@.
This commit is contained in:
ad
parent
06c343ac94
commit
cbbf514e2c
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: kern_cpu.c,v 1.31 2008/05/29 22:33:27 rmind Exp $ */
|
||||
/* $NetBSD: kern_cpu.c,v 1.32 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
|
||||
|
|
@ -57,7 +57,7 @@
|
|||
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.31 2008/05/29 22:33:27 rmind Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.32 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
|
|
@ -113,7 +113,6 @@ mi_cpu_attach(struct cpu_info *ci)
|
|||
__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
|
||||
|
||||
sched_cpuattach(ci);
|
||||
uvm_cpu_attach(ci);
|
||||
|
||||
error = create_idle_lwp(ci);
|
||||
if (error != 0) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: kern_idle.c,v 1.19 2008/05/29 22:33:27 rmind Exp $ */
|
||||
/* $NetBSD: kern_idle.c,v 1.20 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c)2002, 2006, 2007 YAMAMOTO Takashi,
|
||||
|
|
@ -28,7 +28,7 @@
|
|||
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.19 2008/05/29 22:33:27 rmind Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: kern_idle.c,v 1.20 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/cpu.h>
|
||||
|
|
@ -73,23 +73,16 @@ idle_loop(void *dummy)
|
|||
KASSERT(l->l_priority == PRI_IDLE);
|
||||
|
||||
sched_idle();
|
||||
if (sched_curcpu_runnable_p()) {
|
||||
goto schedule;
|
||||
}
|
||||
if (uvm.page_idle_zero) {
|
||||
uvm_pageidlezero();
|
||||
if (sched_curcpu_runnable_p()) {
|
||||
goto schedule;
|
||||
}
|
||||
}
|
||||
if (!sched_curcpu_runnable_p()) {
|
||||
cpu_idle();
|
||||
if (!sched_curcpu_runnable_p() &&
|
||||
!ci->ci_want_resched) {
|
||||
continue;
|
||||
uvm_pageidlezero();
|
||||
if (!sched_curcpu_runnable_p()) {
|
||||
cpu_idle();
|
||||
if (!sched_curcpu_runnable_p() &&
|
||||
!ci->ci_want_resched) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
schedule:
|
||||
KASSERT(l->l_mutex == l->l_cpu->ci_schedstate.spc_lwplock);
|
||||
lwp_lock(l);
|
||||
mi_switch(l);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: subr_autoconf.c,v 1.151 2008/05/27 17:50:03 ad Exp $ */
|
||||
/* $NetBSD: subr_autoconf.c,v 1.152 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1996, 2000 Christopher G. Demetriou
|
||||
|
|
@ -77,7 +77,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: subr_autoconf.c,v 1.151 2008/05/27 17:50:03 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: subr_autoconf.c,v 1.152 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include "opt_ddb.h"
|
||||
#include "drvctl.h"
|
||||
|
|
@ -412,6 +412,8 @@ void
|
|||
configure(void)
|
||||
{
|
||||
extern void ssp_init(void);
|
||||
CPU_INFO_ITERATOR cii;
|
||||
struct cpu_info *ci;
|
||||
int i, s;
|
||||
|
||||
/* Initialize data structures. */
|
||||
|
|
@ -454,6 +456,9 @@ configure(void)
|
|||
splx(s);
|
||||
|
||||
/* Boot the secondary processors. */
|
||||
for (CPU_INFO_FOREACH(cii, ci)) {
|
||||
uvm_cpu_attach(ci);
|
||||
}
|
||||
mp_online = true;
|
||||
#if defined(MULTIPROCESSOR)
|
||||
cpu_boot_secondary_processors();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm.h,v 1.53 2008/01/02 11:49:15 ad Exp $ */
|
||||
/* $NetBSD: uvm.h,v 1.54 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
*
|
||||
|
|
@ -73,6 +73,19 @@
|
|||
|
||||
struct workqueue;
|
||||
|
||||
/*
|
||||
* per-cpu data
|
||||
*/
|
||||
|
||||
struct uvm_cpu {
|
||||
struct pgfreelist page_free[VM_NFREELIST]; /* unallocated pages */
|
||||
int page_free_nextcolor; /* next color to allocate from */
|
||||
int page_idlezero_next; /* which color to zero next */
|
||||
bool page_idle_zero; /* TRUE if we should try to zero
|
||||
pages in the idle loop */
|
||||
int pages[PGFL_NQUEUES]; /* total of pages in page_free */
|
||||
};
|
||||
|
||||
/*
|
||||
* uvm structure (vm global state: collected in one structure for ease
|
||||
* of reference...)
|
||||
|
|
@ -83,10 +96,7 @@ struct uvm {
|
|||
|
||||
/* vm_page queues */
|
||||
struct pgfreelist page_free[VM_NFREELIST]; /* unallocated pages */
|
||||
int page_free_nextcolor; /* next color to allocate from */
|
||||
bool page_init_done; /* TRUE if uvm_page_init() finished */
|
||||
bool page_idle_zero; /* TRUE if we should try to zero
|
||||
pages in the idle loop */
|
||||
|
||||
/* page daemon trigger */
|
||||
int pagedaemon; /* daemon sleeps on this */
|
||||
|
|
@ -108,6 +118,9 @@ struct uvm {
|
|||
kcondvar_t scheduler_cv;
|
||||
bool scheduler_kicked;
|
||||
int swapout_enabled;
|
||||
|
||||
/* per-cpu data */
|
||||
struct uvm_cpu cpus[MAXCPUS];
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_extern.h,v 1.145 2008/02/29 20:35:23 yamt Exp $ */
|
||||
/* $NetBSD: uvm_extern.h,v 1.146 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
*
|
||||
|
|
@ -326,6 +326,8 @@ struct uvmexp {
|
|||
aborted */
|
||||
int colorhit; /* pagealloc where we got optimal color */
|
||||
int colormiss; /* pagealloc where we didn't */
|
||||
int cpuhit; /* pagealloc where we allocated locally */
|
||||
int cpumiss; /* pagealloc where we didn't */
|
||||
|
||||
/* fault subcounters. XXX: should be 64-bit counters */
|
||||
int fltnoram; /* number of times fault was out of ram */
|
||||
|
|
@ -393,8 +395,8 @@ struct uvmexp_sysctl {
|
|||
int64_t swpgonly;
|
||||
int64_t nswget;
|
||||
int64_t unused1; /* used to be nanon */
|
||||
int64_t unused2; /* used to be nanonneeded */
|
||||
int64_t unused3; /* used to be nfreeanon */
|
||||
int64_t cpuhit;
|
||||
int64_t cpumiss;
|
||||
int64_t faults;
|
||||
int64_t traps;
|
||||
int64_t intrs;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_glue.c,v 1.127 2008/05/31 21:26:01 ad Exp $ */
|
||||
/* $NetBSD: uvm_glue.c,v 1.128 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1997 Charles D. Cranor and Washington University.
|
||||
|
|
@ -67,7 +67,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.127 2008/05/31 21:26:01 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.128 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include "opt_coredump.h"
|
||||
#include "opt_kgdb.h"
|
||||
|
|
@ -97,11 +97,6 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.127 2008/05/31 21:26:01 ad Exp $");
|
|||
static void uvm_swapout(struct lwp *);
|
||||
static int uarea_swapin(vaddr_t);
|
||||
|
||||
#define UVM_NUAREA_HIWAT 20
|
||||
#define UVM_NUAREA_LOWAT 16
|
||||
|
||||
#define UAREA_NEXTFREE(uarea) (*(vaddr_t *)(UAREA_TO_USER(uarea)))
|
||||
|
||||
/*
|
||||
* XXXCDC: do these really belong here?
|
||||
*/
|
||||
|
|
@ -276,16 +271,6 @@ uvm_lwp_fork(struct lwp *l1, struct lwp *l2, void *stack, size_t stacksize,
|
|||
cpu_lwp_fork(l1, l2, stack, stacksize, func, arg);
|
||||
}
|
||||
|
||||
/*
|
||||
* uvm_cpu_attach: initialize per-CPU data structures.
|
||||
*/
|
||||
|
||||
void
|
||||
uvm_cpu_attach(struct cpu_info *ci)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
static int
|
||||
uarea_swapin(vaddr_t addr)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_meter.c,v 1.48 2008/04/24 15:35:31 ad Exp $ */
|
||||
/* $NetBSD: uvm_meter.c,v 1.49 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1997 Charles D. Cranor and Washington University.
|
||||
|
|
@ -41,7 +41,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_meter.c,v 1.48 2008/04/24 15:35:31 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_meter.c,v 1.49 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/proc.h>
|
||||
|
|
@ -243,6 +243,8 @@ sysctl_vm_uvmexp2(SYSCTLFN_ARGS)
|
|||
u.execpages = uvmexp.execpages;
|
||||
u.colorhit = uvmexp.colorhit;
|
||||
u.colormiss = uvmexp.colormiss;
|
||||
u.cpuhit = uvmexp.cpuhit;
|
||||
u.cpumiss = uvmexp.cpumiss;
|
||||
|
||||
node = *rnode;
|
||||
node.sysctl_data = &u;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_page.c,v 1.132 2008/06/02 11:11:14 ad Exp $ */
|
||||
/* $NetBSD: uvm_page.c,v 1.133 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1997 Charles D. Cranor and Washington University.
|
||||
|
|
@ -71,7 +71,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.132 2008/06/02 11:11:14 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.133 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include "opt_uvmhist.h"
|
||||
#include "opt_readahead.h"
|
||||
|
|
@ -84,6 +84,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.132 2008/06/02 11:11:14 ad Exp $");
|
|||
#include <sys/vnode.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/atomic.h>
|
||||
#include <sys/cpu.h>
|
||||
|
||||
#include <uvm/uvm.h>
|
||||
#include <uvm/uvm_pdpolicy.h>
|
||||
|
|
@ -219,9 +220,9 @@ uvm_pageinsert_after(struct vm_page *pg, struct vm_page *where)
|
|||
}
|
||||
|
||||
if (where)
|
||||
TAILQ_INSERT_AFTER(&uobj->memq, where, pg, listq);
|
||||
TAILQ_INSERT_AFTER(&uobj->memq, where, pg, listq.queue);
|
||||
else
|
||||
TAILQ_INSERT_TAIL(&uobj->memq, pg, listq);
|
||||
TAILQ_INSERT_TAIL(&uobj->memq, pg, listq.queue);
|
||||
pg->flags |= PG_TABLED;
|
||||
uobj->uo_npages++;
|
||||
}
|
||||
|
|
@ -275,7 +276,7 @@ uvm_pageremove(struct vm_page *pg)
|
|||
|
||||
/* object should be locked */
|
||||
uobj->uo_npages--;
|
||||
TAILQ_REMOVE(&uobj->memq, pg, listq);
|
||||
TAILQ_REMOVE(&uobj->memq, pg, listq.queue);
|
||||
pg->flags &= ~PG_TABLED;
|
||||
pg->uobject = NULL;
|
||||
}
|
||||
|
|
@ -287,7 +288,7 @@ uvm_page_init_buckets(struct pgfreelist *pgfl)
|
|||
|
||||
for (color = 0; color < uvmexp.ncolors; color++) {
|
||||
for (i = 0; i < PGFL_NQUEUES; i++) {
|
||||
TAILQ_INIT(&pgfl->pgfl_buckets[color].pgfl_queues[i]);
|
||||
LIST_INIT(&pgfl->pgfl_buckets[color].pgfl_queues[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -302,18 +303,22 @@ void
|
|||
uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
|
||||
{
|
||||
vsize_t freepages, pagecount, bucketcount, n;
|
||||
struct pgflbucket *bucketarray;
|
||||
struct pgflbucket *bucketarray, *cpuarray;
|
||||
struct vm_page *pagearray;
|
||||
int lcv;
|
||||
u_int i;
|
||||
paddr_t paddr;
|
||||
|
||||
KASSERT(ncpu <= 1);
|
||||
KASSERT(sizeof(pagearray->offset) >= sizeof(struct uvm_cpu *));
|
||||
|
||||
/*
|
||||
* init the page queues and page queue locks, except the free
|
||||
* list; we allocate that later (with the initial vm_page
|
||||
* structures).
|
||||
*/
|
||||
|
||||
curcpu()->ci_data.cpu_uvm = &uvm.cpus[0];
|
||||
uvmpdpol_init();
|
||||
mutex_init(&uvm_pageqlock, MUTEX_DRIVER, IPL_NONE);
|
||||
mutex_init(&uvm_fpageqlock, MUTEX_DRIVER, IPL_VM);
|
||||
|
|
@ -388,14 +393,18 @@ uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
|
|||
(PAGE_SIZE + sizeof(struct vm_page));
|
||||
|
||||
bucketarray = (void *)uvm_pageboot_alloc((bucketcount *
|
||||
sizeof(struct pgflbucket)) + (pagecount *
|
||||
sizeof(struct pgflbucket) * 2) + (pagecount *
|
||||
sizeof(struct vm_page)));
|
||||
pagearray = (struct vm_page *)(bucketarray + bucketcount);
|
||||
cpuarray = bucketarray + bucketcount;
|
||||
pagearray = (struct vm_page *)(bucketarray + bucketcount * 2);
|
||||
|
||||
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
|
||||
uvm.page_free[lcv].pgfl_buckets =
|
||||
(bucketarray + (lcv * uvmexp.ncolors));
|
||||
uvm_page_init_buckets(&uvm.page_free[lcv]);
|
||||
uvm.cpus[0].page_free[lcv].pgfl_buckets =
|
||||
(cpuarray + (lcv * uvmexp.ncolors));
|
||||
uvm_page_init_buckets(&uvm.cpus[0].page_free[lcv]);
|
||||
}
|
||||
memset(pagearray, 0, pagecount * sizeof(struct vm_page));
|
||||
|
||||
|
|
@ -455,7 +464,7 @@ uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
|
|||
* determine if we should zero pages in the idle loop.
|
||||
*/
|
||||
|
||||
uvm.page_idle_zero = vm_page_zero_enable;
|
||||
uvm.cpus[0].page_idle_zero = vm_page_zero_enable;
|
||||
|
||||
/*
|
||||
* done!
|
||||
|
|
@ -933,11 +942,12 @@ uvm_page_rehash(void)
|
|||
void
|
||||
uvm_page_recolor(int newncolors)
|
||||
{
|
||||
struct pgflbucket *bucketarray, *oldbucketarray;
|
||||
struct pgfreelist pgfl;
|
||||
struct pgflbucket *bucketarray, *cpuarray, *oldbucketarray;
|
||||
struct pgfreelist gpgfl, pgfl;
|
||||
struct vm_page *pg;
|
||||
vsize_t bucketcount;
|
||||
int lcv, color, i, ocolors;
|
||||
struct uvm_cpu *ucpu;
|
||||
|
||||
if (newncolors <= uvmexp.ncolors)
|
||||
return;
|
||||
|
|
@ -948,8 +958,9 @@ uvm_page_recolor(int newncolors)
|
|||
}
|
||||
|
||||
bucketcount = newncolors * VM_NFREELIST;
|
||||
bucketarray = malloc(bucketcount * sizeof(struct pgflbucket),
|
||||
bucketarray = malloc(bucketcount * sizeof(struct pgflbucket) * 2,
|
||||
M_VMPAGE, M_NOWAIT);
|
||||
cpuarray = bucketarray + bucketcount;
|
||||
if (bucketarray == NULL) {
|
||||
printf("WARNING: unable to allocate %ld page color buckets\n",
|
||||
(long) bucketcount);
|
||||
|
|
@ -971,24 +982,30 @@ uvm_page_recolor(int newncolors)
|
|||
uvmexp.ncolors = newncolors;
|
||||
uvmexp.colormask = uvmexp.ncolors - 1;
|
||||
|
||||
ucpu = curcpu()->ci_data.cpu_uvm;
|
||||
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
|
||||
pgfl.pgfl_buckets = (bucketarray + (lcv * newncolors));
|
||||
gpgfl.pgfl_buckets = (bucketarray + (lcv * newncolors));
|
||||
pgfl.pgfl_buckets = (cpuarray + (lcv * uvmexp.ncolors));
|
||||
uvm_page_init_buckets(&gpgfl);
|
||||
uvm_page_init_buckets(&pgfl);
|
||||
for (color = 0; color < ocolors; color++) {
|
||||
for (i = 0; i < PGFL_NQUEUES; i++) {
|
||||
while ((pg = TAILQ_FIRST(&uvm.page_free[
|
||||
while ((pg = LIST_FIRST(&uvm.page_free[
|
||||
lcv].pgfl_buckets[color].pgfl_queues[i]))
|
||||
!= NULL) {
|
||||
TAILQ_REMOVE(&uvm.page_free[
|
||||
lcv].pgfl_buckets[
|
||||
color].pgfl_queues[i], pg, pageq);
|
||||
TAILQ_INSERT_TAIL(&pgfl.pgfl_buckets[
|
||||
LIST_REMOVE(pg, pageq.list); /* global */
|
||||
LIST_REMOVE(pg, listq.list); /* cpu */
|
||||
LIST_INSERT_HEAD(&gpgfl.pgfl_buckets[
|
||||
VM_PGCOLOR_BUCKET(pg)].pgfl_queues[
|
||||
i], pg, pageq);
|
||||
i], pg, pageq.list);
|
||||
LIST_INSERT_HEAD(&pgfl.pgfl_buckets[
|
||||
VM_PGCOLOR_BUCKET(pg)].pgfl_queues[
|
||||
i], pg, listq.list);
|
||||
}
|
||||
}
|
||||
}
|
||||
uvm.page_free[lcv].pgfl_buckets = pgfl.pgfl_buckets;
|
||||
uvm.page_free[lcv].pgfl_buckets = gpgfl.pgfl_buckets;
|
||||
ucpu->page_free[lcv].pgfl_buckets = pgfl.pgfl_buckets;
|
||||
}
|
||||
|
||||
if (have_recolored_pages) {
|
||||
|
|
@ -1001,35 +1018,92 @@ uvm_page_recolor(int newncolors)
|
|||
mutex_spin_exit(&uvm_fpageqlock);
|
||||
}
|
||||
|
||||
/*
|
||||
* uvm_cpu_attach: initialize per-CPU data structures.
|
||||
*/
|
||||
|
||||
void
|
||||
uvm_cpu_attach(struct cpu_info *ci)
|
||||
{
|
||||
struct pgflbucket *bucketarray;
|
||||
struct pgfreelist pgfl;
|
||||
struct uvm_cpu *ucpu;
|
||||
vsize_t bucketcount;
|
||||
int lcv;
|
||||
|
||||
if (CPU_IS_PRIMARY(ci)) {
|
||||
/* Already done in uvm_page_init(). */
|
||||
return;
|
||||
}
|
||||
|
||||
bucketcount = uvmexp.ncolors * VM_NFREELIST;
|
||||
bucketarray = malloc(bucketcount * sizeof(struct pgflbucket),
|
||||
M_VMPAGE, M_WAITOK);
|
||||
ucpu = &uvm.cpus[cpu_index(ci)];
|
||||
ci->ci_data.cpu_uvm = ucpu;
|
||||
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
|
||||
pgfl.pgfl_buckets = (bucketarray + (lcv * uvmexp.ncolors));
|
||||
uvm_page_init_buckets(&pgfl);
|
||||
ucpu->page_free[lcv].pgfl_buckets = pgfl.pgfl_buckets;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* uvm_pagealloc_pgfl: helper routine for uvm_pagealloc_strat
|
||||
*/
|
||||
|
||||
static struct vm_page *
|
||||
uvm_pagealloc_pgfl(struct pgfreelist *pgfl, int try1, int try2,
|
||||
uvm_pagealloc_pgfl(struct uvm_cpu *ucpu, int flist, int try1, int try2,
|
||||
int *trycolorp)
|
||||
{
|
||||
struct pglist *freeq;
|
||||
struct pgflist *freeq;
|
||||
struct vm_page *pg;
|
||||
int color, trycolor = *trycolorp;
|
||||
struct pgfreelist *gpgfl, *pgfl;
|
||||
|
||||
KASSERT(mutex_owned(&uvm_fpageqlock));
|
||||
|
||||
color = trycolor;
|
||||
cpu = false;
|
||||
pgfl = &ucpu->page_free[flist];
|
||||
gpgfl = &uvm.page_free[flist];
|
||||
do {
|
||||
if ((pg = TAILQ_FIRST((freeq =
|
||||
&pgfl->pgfl_buckets[color].pgfl_queues[try1]))) != NULL)
|
||||
/* cpu, try1 */
|
||||
if ((pg = LIST_FIRST((freeq =
|
||||
&pgfl->pgfl_buckets[color].pgfl_queues[try1]))) != NULL) {
|
||||
VM_FREE_PAGE_TO_CPU(pg)->pages[try1]--;
|
||||
uvmexp.cpuhit++;
|
||||
goto gotit;
|
||||
if ((pg = TAILQ_FIRST((freeq =
|
||||
&pgfl->pgfl_buckets[color].pgfl_queues[try2]))) != NULL)
|
||||
}
|
||||
/* global, try1 */
|
||||
if ((pg = LIST_FIRST((freeq =
|
||||
&gpgfl->pgfl_buckets[color].pgfl_queues[try1]))) != NULL) {
|
||||
VM_FREE_PAGE_TO_CPU(pg)->pages[try1]--;
|
||||
uvmexp.cpumiss++;
|
||||
goto gotit;
|
||||
}
|
||||
/* cpu, try2 */
|
||||
if ((pg = LIST_FIRST((freeq =
|
||||
&pgfl->pgfl_buckets[color].pgfl_queues[try2]))) != NULL) {
|
||||
VM_FREE_PAGE_TO_CPU(pg)->pages[try2]--;
|
||||
uvmexp.cpuhit++;
|
||||
goto gotit;
|
||||
}
|
||||
/* global, try2 */
|
||||
if ((pg = LIST_FIRST((freeq =
|
||||
&gpgfl->pgfl_buckets[color].pgfl_queues[try2]))) != NULL) {
|
||||
VM_FREE_PAGE_TO_CPU(pg)->pages[try2]--;
|
||||
uvmexp.cpumiss++;
|
||||
goto gotit;
|
||||
}
|
||||
color = (color + 1) & uvmexp.colormask;
|
||||
} while (color != trycolor);
|
||||
|
||||
return (NULL);
|
||||
|
||||
gotit:
|
||||
TAILQ_REMOVE(freeq, pg, pageq);
|
||||
LIST_REMOVE(pg, pageq.list); /* global list */
|
||||
LIST_REMOVE(pg, listq.list); /* per-cpu list */
|
||||
uvmexp.free--;
|
||||
|
||||
/* update zero'd page count */
|
||||
|
|
@ -1051,7 +1125,7 @@ uvm_pagealloc_pgfl(struct pgfreelist *pgfl, int try1, int try2,
|
|||
*
|
||||
* => return null if no pages free
|
||||
* => wake up pagedaemon if number of free pages drops below low water mark
|
||||
* => if obj != NULL, obj must be locked (to put in hash)
|
||||
* => if obj != NULL, obj must be locked (to put in obj's tree)
|
||||
* => if anon != NULL, anon must be locked (to put in anon)
|
||||
* => only one of obj or anon can be non-null
|
||||
* => caller must activate/deactivate page if it is not wired.
|
||||
|
|
@ -1069,6 +1143,7 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
|
|||
int flags, int strat, int free_list)
|
||||
{
|
||||
int lcv, try1, try2, zeroit = 0, color;
|
||||
struct uvm_cpu *ucpu;
|
||||
struct vm_page *pg;
|
||||
bool use_reserve;
|
||||
|
||||
|
|
@ -1084,11 +1159,11 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
|
|||
* This implements a global round-robin page coloring
|
||||
* algorithm.
|
||||
*
|
||||
* XXXJRT: Should we make the `nextcolor' per-CPU?
|
||||
* XXXJRT: What about virtually-indexed caches?
|
||||
*/
|
||||
|
||||
color = uvm.page_free_nextcolor;
|
||||
ucpu = curcpu()->ci_data.cpu_uvm;
|
||||
color = ucpu->page_free_nextcolor;
|
||||
|
||||
/*
|
||||
* check to see if we need to generate some free pages waking
|
||||
|
|
@ -1134,7 +1209,7 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
|
|||
case UVM_PGA_STRAT_NORMAL:
|
||||
/* Check all freelists in descending priority order. */
|
||||
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
|
||||
pg = uvm_pagealloc_pgfl(&uvm.page_free[lcv],
|
||||
pg = uvm_pagealloc_pgfl(ucpu, lcv,
|
||||
try1, try2, &color);
|
||||
if (pg != NULL)
|
||||
goto gotit;
|
||||
|
|
@ -1147,7 +1222,7 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
|
|||
case UVM_PGA_STRAT_FALLBACK:
|
||||
/* Attempt to allocate from the specified free list. */
|
||||
KASSERT(free_list >= 0 && free_list < VM_NFREELIST);
|
||||
pg = uvm_pagealloc_pgfl(&uvm.page_free[free_list],
|
||||
pg = uvm_pagealloc_pgfl(ucpu, free_list,
|
||||
try1, try2, &color);
|
||||
if (pg != NULL)
|
||||
goto gotit;
|
||||
|
|
@ -1172,7 +1247,7 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
|
|||
* the next color accordingly.
|
||||
*/
|
||||
|
||||
uvm.page_free_nextcolor = (color + 1) & uvmexp.colormask;
|
||||
ucpu->page_free_nextcolor = (color + 1) & uvmexp.colormask;
|
||||
|
||||
/*
|
||||
* update allocation statistics and remember if we have to
|
||||
|
|
@ -1187,6 +1262,9 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
|
|||
uvmexp.pga_zeromiss++;
|
||||
zeroit = 1;
|
||||
}
|
||||
if (ucpu->pages[PGFL_ZEROS] < ucpu->pages[PGFL_UNKNOWN]) {
|
||||
ucpu->page_idle_zero = vm_page_zero_enable;
|
||||
}
|
||||
}
|
||||
mutex_spin_exit(&uvm_fpageqlock);
|
||||
|
||||
|
|
@ -1317,7 +1395,7 @@ uvm_pagezerocheck(struct vm_page *pg)
|
|||
/*
|
||||
* uvm_pagefree: free page
|
||||
*
|
||||
* => erase page's identity (i.e. remove from hash/object)
|
||||
* => erase page's identity (i.e. remove from object)
|
||||
* => put page on free list
|
||||
* => caller must lock owning object (either anon or uvm_object)
|
||||
* => caller must lock page queues
|
||||
|
|
@ -1327,7 +1405,9 @@ uvm_pagezerocheck(struct vm_page *pg)
|
|||
void
|
||||
uvm_pagefree(struct vm_page *pg)
|
||||
{
|
||||
struct pglist *pgfl;
|
||||
struct pgflist *pgfl;
|
||||
struct uvm_cpu *ucpu;
|
||||
int index, color, queue;
|
||||
bool iszero;
|
||||
|
||||
#ifdef DEBUG
|
||||
|
|
@ -1421,14 +1501,13 @@ uvm_pagefree(struct vm_page *pg)
|
|||
*/
|
||||
|
||||
iszero = (pg->flags & PG_ZERO);
|
||||
pgfl = &uvm.page_free[uvm_page_lookup_freelist(pg)].
|
||||
pgfl_buckets[VM_PGCOLOR_BUCKET(pg)].
|
||||
pgfl_queues[iszero ? PGFL_ZEROS : PGFL_UNKNOWN];
|
||||
index = uvm_page_lookup_freelist(pg);
|
||||
color = VM_PGCOLOR_BUCKET(pg);
|
||||
queue = (iszero ? PGFL_ZEROS : PGFL_UNKNOWN);
|
||||
|
||||
pg->pqflags = PQ_FREE;
|
||||
#ifdef DEBUG
|
||||
pg->uobject = (void *)0xdeadbeef;
|
||||
pg->offset = 0xdeadbeef;
|
||||
pg->uanon = (void *)0xdeadbeef;
|
||||
#endif
|
||||
|
||||
|
|
@ -1439,13 +1518,24 @@ uvm_pagefree(struct vm_page *pg)
|
|||
uvm_pagezerocheck(pg);
|
||||
#endif /* DEBUG */
|
||||
|
||||
TAILQ_INSERT_HEAD(pgfl, pg, pageq);
|
||||
uvmexp.free++;
|
||||
if (iszero)
|
||||
uvmexp.zeropages++;
|
||||
|
||||
if (uvmexp.zeropages < UVM_PAGEZERO_LOWAT)
|
||||
uvm.page_idle_zero = vm_page_zero_enable;
|
||||
/* global list */
|
||||
pgfl = &uvm.page_free[index].pgfl_buckets[color].pgfl_queues[queue];
|
||||
LIST_INSERT_HEAD(pgfl, pg, pageq.list);
|
||||
uvmexp.free++;
|
||||
if (iszero) {
|
||||
uvmexp.zeropages++;
|
||||
}
|
||||
|
||||
/* per-cpu list */
|
||||
ucpu = curcpu()->ci_data.cpu_uvm;
|
||||
pg->offset = (uintptr_t)ucpu;
|
||||
pgfl = &ucpu->page_free[index].pgfl_buckets[color].pgfl_queues[queue];
|
||||
LIST_INSERT_HEAD(pgfl, pg, listq.list);
|
||||
ucpu->pages[queue]++;
|
||||
if (ucpu->pages[PGFL_ZEROS] < ucpu->pages[PGFL_UNKNOWN]) {
|
||||
ucpu->page_idle_zero = vm_page_zero_enable;
|
||||
}
|
||||
|
||||
mutex_spin_exit(&uvm_fpageqlock);
|
||||
}
|
||||
|
|
@ -1564,44 +1654,39 @@ uvm_page_own(struct vm_page *pg, const char *tag)
|
|||
* on the CPU cache.
|
||||
* => we loop until we either reach the target or there is a lwp ready
|
||||
* to run, or MD code detects a reason to break early.
|
||||
* => we only allow one CPU at a time to zero in the idle loop, to try
|
||||
* and avoid swamping the system bus.
|
||||
*/
|
||||
void
|
||||
uvm_pageidlezero(void)
|
||||
{
|
||||
struct vm_page *pg;
|
||||
struct pgfreelist *pgfl;
|
||||
int free_list, firstbucket;
|
||||
static int nextbucket;
|
||||
static __cpu_simple_lock_t idlezero_lock = __SIMPLELOCK_UNLOCKED;
|
||||
struct pgfreelist *pgfl, *gpgfl;
|
||||
struct uvm_cpu *ucpu;
|
||||
int free_list, firstbucket, nextbucket;
|
||||
|
||||
if (!__cpu_simple_lock_try(&idlezero_lock)) {
|
||||
ucpu = curcpu()->ci_data.cpu_uvm;
|
||||
if (!ucpu->page_idle_zero ||
|
||||
ucpu->pages[PGFL_UNKNOWN] < uvmexp.ncolors) {
|
||||
ucpu->page_idle_zero = false;
|
||||
return;
|
||||
}
|
||||
if (!mutex_tryenter(&uvm_fpageqlock)) {
|
||||
__cpu_simple_unlock(&idlezero_lock);
|
||||
return;
|
||||
}
|
||||
firstbucket = nextbucket;
|
||||
mutex_enter(&uvm_fpageqlock);
|
||||
firstbucket = ucpu->page_free_nextcolor;
|
||||
nextbucket = firstbucket;
|
||||
do {
|
||||
if (sched_curcpu_runnable_p()) {
|
||||
goto quit;
|
||||
}
|
||||
if (uvmexp.zeropages >= UVM_PAGEZERO_HIWAT) {
|
||||
uvm.page_idle_zero = false;
|
||||
goto quit;
|
||||
break;
|
||||
}
|
||||
for (free_list = 0; free_list < VM_NFREELIST; free_list++) {
|
||||
pgfl = &uvm.page_free[free_list];
|
||||
while ((pg = TAILQ_FIRST(&pgfl->pgfl_buckets[
|
||||
pgfl = &ucpu->page_free[free_list];
|
||||
gpgfl = &uvm.page_free[free_list];
|
||||
while ((pg = LIST_FIRST(&pgfl->pgfl_buckets[
|
||||
nextbucket].pgfl_queues[PGFL_UNKNOWN])) != NULL) {
|
||||
if (sched_curcpu_runnable_p()) {
|
||||
goto quit;
|
||||
}
|
||||
TAILQ_REMOVE(&pgfl->pgfl_buckets[
|
||||
nextbucket].pgfl_queues[PGFL_UNKNOWN],
|
||||
pg, pageq);
|
||||
LIST_REMOVE(pg, pageq.list); /* global list */
|
||||
LIST_REMOVE(pg, listq.list); /* per-cpu list */
|
||||
ucpu->pages[PGFL_UNKNOWN]--;
|
||||
uvmexp.free--;
|
||||
mutex_spin_exit(&uvm_fpageqlock);
|
||||
#ifdef PMAP_PAGEIDLEZERO
|
||||
|
|
@ -1615,9 +1700,13 @@ uvm_pageidlezero(void)
|
|||
*/
|
||||
|
||||
mutex_spin_enter(&uvm_fpageqlock);
|
||||
TAILQ_INSERT_HEAD(&pgfl->pgfl_buckets[
|
||||
LIST_INSERT_HEAD(&gpgfl->pgfl_buckets[
|
||||
nextbucket].pgfl_queues[
|
||||
PGFL_UNKNOWN], pg, pageq);
|
||||
PGFL_UNKNOWN], pg, pageq.list);
|
||||
LIST_INSERT_HEAD(&pgfl->pgfl_buckets[
|
||||
nextbucket].pgfl_queues[
|
||||
PGFL_UNKNOWN], pg, listq.list);
|
||||
ucpu->pages[PGFL_UNKNOWN]++;
|
||||
uvmexp.free++;
|
||||
uvmexp.zeroaborts++;
|
||||
goto quit;
|
||||
|
|
@ -1628,18 +1717,25 @@ uvm_pageidlezero(void)
|
|||
pg->flags |= PG_ZERO;
|
||||
|
||||
mutex_spin_enter(&uvm_fpageqlock);
|
||||
TAILQ_INSERT_HEAD(&pgfl->pgfl_buckets[
|
||||
LIST_INSERT_HEAD(&gpgfl->pgfl_buckets[
|
||||
nextbucket].pgfl_queues[PGFL_ZEROS],
|
||||
pg, pageq);
|
||||
pg, pageq.list);
|
||||
LIST_INSERT_HEAD(&pgfl->pgfl_buckets[
|
||||
nextbucket].pgfl_queues[PGFL_ZEROS],
|
||||
pg, listq.list);
|
||||
ucpu->pages[PGFL_ZEROS]++;
|
||||
uvmexp.free++;
|
||||
uvmexp.zeropages++;
|
||||
}
|
||||
}
|
||||
if (ucpu->pages[PGFL_UNKNOWN] < uvmexp.ncolors) {
|
||||
break;
|
||||
}
|
||||
nextbucket = (nextbucket + 1) & uvmexp.colormask;
|
||||
} while (nextbucket != firstbucket);
|
||||
quit:
|
||||
ucpu->page_idle_zero = false;
|
||||
quit:
|
||||
mutex_spin_exit(&uvm_fpageqlock);
|
||||
__cpu_simple_unlock(&idlezero_lock);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_page.h,v 1.53 2008/06/02 11:11:14 ad Exp $ */
|
||||
/* $NetBSD: uvm_page.h,v 1.54 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 1997 Charles D. Cranor and Washington University.
|
||||
|
|
@ -118,10 +118,17 @@
|
|||
#include <uvm/uvm_pglist.h>
|
||||
|
||||
struct vm_page {
|
||||
TAILQ_ENTRY(vm_page) pageq; /* queue info for FIFO
|
||||
* queue or free list (P) */
|
||||
TAILQ_ENTRY(vm_page) hashq; /* hash table links (O)*/
|
||||
TAILQ_ENTRY(vm_page) listq; /* pages in same object (O)*/
|
||||
|
||||
union {
|
||||
TAILQ_ENTRY(vm_page) queue;
|
||||
LIST_ENTRY(vm_page) list;
|
||||
} pageq; /* queue info for FIFO
|
||||
* queue or free list (P) */
|
||||
union {
|
||||
TAILQ_ENTRY(vm_page) queue;
|
||||
LIST_ENTRY(vm_page) list;
|
||||
} listq; /* pages in same object (O)*/
|
||||
|
||||
struct vm_anon *uanon; /* anon (O,P) */
|
||||
struct uvm_object *uobject; /* object (O,P) */
|
||||
|
|
@ -291,9 +298,6 @@ static int vm_physseg_find(paddr_t, int *);
|
|||
#define uvm_pagehash(obj,off) \
|
||||
(((unsigned long)obj+(unsigned long)atop(off)) & uvm.page_hashmask)
|
||||
|
||||
#define UVM_PAGEZERO_LOWAT (uvmexp.free >> 1)
|
||||
#define UVM_PAGEZERO_HIWAT (uvmexp.free * 10 / 9)
|
||||
|
||||
#define VM_PAGE_TO_PHYS(entry) ((entry)->phys_addr)
|
||||
|
||||
/*
|
||||
|
|
@ -403,6 +407,7 @@ PHYS_TO_VM_PAGE(paddr_t pa)
|
|||
}
|
||||
|
||||
#define VM_PAGE_IS_FREE(entry) ((entry)->pqflags & PQ_FREE)
|
||||
#define VM_FREE_PAGE_TO_CPU(pg) ((struct uvm_cpu *)((uintptr_t)pg->offset))
|
||||
|
||||
#ifdef DEBUG
|
||||
void uvm_pagezerocheck(struct vm_page *);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: uvm_pglist.c,v 1.41 2008/06/02 12:24:16 ad Exp $ */
|
||||
/* $NetBSD: uvm_pglist.c,v 1.42 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 1997 The NetBSD Foundation, Inc.
|
||||
|
|
@ -35,7 +35,7 @@
|
|||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.41 2008/06/02 12:24:16 ad Exp $");
|
||||
__KERNEL_RCSID(0, "$NetBSD: uvm_pglist.c,v 1.42 2008/06/04 12:45:28 ad Exp $");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
|
|
@ -96,26 +96,27 @@ uvm_pglist_add(struct vm_page *pg, struct pglist *rlist)
|
|||
color = VM_PGCOLOR_BUCKET(pg);
|
||||
pgflidx = (pg->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN;
|
||||
#ifdef DEBUG
|
||||
for (tp = TAILQ_FIRST(&uvm.page_free[
|
||||
for (tp = LIST_FIRST(&uvm.page_free[
|
||||
free_list].pgfl_buckets[color].pgfl_queues[pgflidx]);
|
||||
tp != NULL;
|
||||
tp = TAILQ_NEXT(tp, pageq)) {
|
||||
tp = LIST_NEXT(tp, pageq.list)) {
|
||||
if (tp == pg)
|
||||
break;
|
||||
}
|
||||
if (tp == NULL)
|
||||
panic("uvm_pglistalloc: page not on freelist");
|
||||
#endif
|
||||
TAILQ_REMOVE(&uvm.page_free[free_list].pgfl_buckets[
|
||||
color].pgfl_queues[pgflidx], pg, pageq);
|
||||
LIST_REMOVE(pg, pageq.list); /* global */
|
||||
LIST_REMOVE(pg, listq.list); /* cpu */
|
||||
uvmexp.free--;
|
||||
if (pg->flags & PG_ZERO)
|
||||
uvmexp.zeropages--;
|
||||
VM_FREE_PAGE_TO_CPU(pg)->pages[pgflidx]--;
|
||||
pg->flags = PG_CLEAN;
|
||||
pg->pqflags = 0;
|
||||
pg->uobject = NULL;
|
||||
pg->uanon = NULL;
|
||||
TAILQ_INSERT_TAIL(rlist, pg, pageq);
|
||||
TAILQ_INSERT_TAIL(rlist, pg, pageq.queue);
|
||||
STAT_INCR(uvm_pglistalloc_npages);
|
||||
}
|
||||
|
||||
|
|
@ -433,38 +434,45 @@ uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment,
|
|||
void
|
||||
uvm_pglistfree(struct pglist *list)
|
||||
{
|
||||
struct uvm_cpu *ucpu;
|
||||
struct vm_page *pg;
|
||||
int index, color, queue;
|
||||
bool iszero;
|
||||
|
||||
/*
|
||||
* Lock the free list and free each page.
|
||||
*/
|
||||
|
||||
mutex_spin_enter(&uvm_fpageqlock);
|
||||
ucpu = curcpu()->ci_data.cpu_uvm;
|
||||
while ((pg = TAILQ_FIRST(list)) != NULL) {
|
||||
bool iszero;
|
||||
|
||||
KASSERT(!uvmpdpol_pageisqueued_p(pg));
|
||||
TAILQ_REMOVE(list, pg, pageq);
|
||||
TAILQ_REMOVE(list, pg, pageq.queue);
|
||||
iszero = (pg->flags & PG_ZERO);
|
||||
pg->pqflags = PQ_FREE;
|
||||
#ifdef DEBUG
|
||||
pg->uobject = (void *)0xdeadbeef;
|
||||
pg->offset = 0xdeadbeef;
|
||||
pg->uanon = (void *)0xdeadbeef;
|
||||
#endif /* DEBUG */
|
||||
#ifdef DEBUG
|
||||
if (iszero)
|
||||
uvm_pagezerocheck(pg);
|
||||
#endif /* DEBUG */
|
||||
TAILQ_INSERT_HEAD(&uvm.page_free[uvm_page_lookup_freelist(pg)].
|
||||
pgfl_buckets[VM_PGCOLOR_BUCKET(pg)].
|
||||
pgfl_queues[iszero ? PGFL_ZEROS : PGFL_UNKNOWN], pg, pageq);
|
||||
index = uvm_page_lookup_freelist(pg);
|
||||
color = VM_PGCOLOR_BUCKET(pg);
|
||||
queue = iszero ? PGFL_ZEROS : PGFL_UNKNOWN;
|
||||
pg->offset = (uintptr_t)ucpu;
|
||||
LIST_INSERT_HEAD(&uvm.page_free[index].pgfl_buckets[color].
|
||||
pgfl_queues[queue], pg, pageq.list);
|
||||
LIST_INSERT_HEAD(&ucpu->page_free[index].pgfl_buckets[color].
|
||||
pgfl_queues[queue], pg, listq.list);
|
||||
uvmexp.free++;
|
||||
if (iszero)
|
||||
uvmexp.zeropages++;
|
||||
if (uvmexp.zeropages < UVM_PAGEZERO_LOWAT)
|
||||
uvm.page_idle_zero = vm_page_zero_enable;
|
||||
ucpu->pages[queue]++;
|
||||
STAT_DECR(uvm_pglistalloc_npages);
|
||||
}
|
||||
if (ucpu->pages[PGFL_ZEROS] < ucpu->pages[PGFL_UNKNOWN])
|
||||
ucpu->page_idle_zero = vm_page_zero_enable;
|
||||
mutex_spin_exit(&uvm_fpageqlock);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* $NetBSD: uvm_pglist.h,v 1.6 2008/04/28 20:24:12 martin Exp $ */
|
||||
/* $NetBSD: uvm_pglist.h,v 1.7 2008/06/04 12:45:28 ad Exp $ */
|
||||
|
||||
/*-
|
||||
* Copyright (c) 2000, 2001 The NetBSD Foundation, Inc.
|
||||
* Copyright (c) 2000, 2001, 2008 The NetBSD Foundation, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This code is derived from software contributed to The NetBSD Foundation
|
||||
|
|
@ -37,6 +37,7 @@
|
|||
* list, etc.
|
||||
*/
|
||||
TAILQ_HEAD(pglist, vm_page);
|
||||
LIST_HEAD(pgflist, vm_page);
|
||||
|
||||
/*
|
||||
* A page free list consists of free pages of unknown contents and free
|
||||
|
|
@ -47,7 +48,7 @@ TAILQ_HEAD(pglist, vm_page);
|
|||
#define PGFL_NQUEUES 2
|
||||
|
||||
struct pgflbucket {
|
||||
struct pglist pgfl_queues[PGFL_NQUEUES];
|
||||
struct pgflist pgfl_queues[PGFL_NQUEUES];
|
||||
};
|
||||
|
||||
struct pgfreelist {
|
||||
|
|
|
|||
Loading…
Reference in New Issue