Changes necessary to implement pre-zero'ing of pages in the idle loop:

- Make page free lists have two actual queues: known-zero pages and
  pages with unknown contents.
- Implement uvm_pageidlezero().  This function attempts to zero up to
  the target number of pages until the target has been reached (currently
  target is `all free pages') or until whichqs becomes non-zero (indicating
  that a process is ready to run).
- Define a new hook for the pmap module for pre-zero'ing pages.  This is
  used to zero the pages using uncached access.  This allows us to zero
  as many pages as we want without polluting the cache.

In order to use this feature, each platform must add the appropropriate
glue in their idle loop.
This commit is contained in:
thorpej 2000-04-24 17:12:00 +00:00
parent b7de2c5900
commit 9ec517a68e
8 changed files with 257 additions and 40 deletions

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm.h,v 1.20 2000/04/10 02:20:06 chs Exp $ */
/* $NetBSD: uvm.h,v 1.21 2000/04/24 17:12:00 thorpej Exp $ */
/*
*
@ -77,13 +77,15 @@
struct uvm {
/* vm_page related parameters */
/* vm_page queues */
struct pglist page_free[VM_NFREELIST]; /* unallocated pages */
struct pgfreelist page_free[VM_NFREELIST]; /* unallocated pages */
struct pglist page_active; /* allocated pages, in use */
struct pglist page_inactive_swp;/* pages inactive (reclaim or free) */
struct pglist page_inactive_obj;/* pages inactive (reclaim or free) */
simple_lock_data_t pageqlock; /* lock for active/inactive page q */
simple_lock_data_t fpageqlock; /* lock for free page q */
boolean_t page_init_done; /* TRUE if uvm_page_init() finished */
boolean_t page_idle_zero; /* TRUE if we should try to zero
pages in the idle loop */
/* page daemon trigger */
int pagedaemon; /* daemon sleeps on this */
struct proc *pagedaemon_proc; /* daemon's pid */

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_extern.h,v 1.39 2000/04/10 00:28:05 thorpej Exp $ */
/* $NetBSD: uvm_extern.h,v 1.40 2000/04/24 17:12:00 thorpej Exp $ */
/*
*
@ -176,6 +176,7 @@ struct uvmexp {
int inactive; /* number of pages that we free'd but may want back */
int paging; /* number of pages in the process of being paged out */
int wired; /* number of wired pages */
int zeropages; /* number of zero'd pages */
int reserve_pagedaemon; /* number of pages reserved for pagedaemon */
int reserve_kernel; /* number of pages reserved for kernel */
@ -212,6 +213,10 @@ struct uvmexp {
int forks; /* forks */
int forks_ppwait; /* forks where parent waits */
int forks_sharevm; /* forks where vmspace is shared */
int pga_zerohit; /* pagealloc where zero wanted and zero
was available */
int pga_zeromiss; /* pagealloc where zero wanted and zero
not available */
/* fault subcounters */
int fltnoram; /* number of times fault was out of ram */

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_map.c,v 1.72 2000/04/16 20:52:29 chs Exp $ */
/* $NetBSD: uvm_map.c,v 1.73 2000/04/24 17:12:00 thorpej Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -3367,8 +3367,11 @@ uvm_page_printit(pg, full, pr)
}
/* cross-verify page queue */
if (pg->pqflags & PQ_FREE)
pgl = &uvm.page_free[uvm_page_lookup_freelist(pg)];
if (pg->pqflags & PQ_FREE) {
int fl = uvm_page_lookup_freelist(pg);
pgl = &uvm.page_free[fl].pgfl_queues[((pg)->flags & PG_ZERO) ?
PGFL_ZEROS : PGFL_UNKNOWN];
}
else if (pg->pqflags & PQ_INACTIVE)
pgl = (pg->pqflags & PQ_SWAPBACKED) ?
&uvm.page_inactive_swp : &uvm.page_inactive_obj;

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_page.c,v 1.33 2000/04/10 00:28:05 thorpej Exp $ */
/* $NetBSD: uvm_page.c,v 1.34 2000/04/24 17:12:01 thorpej Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -93,6 +93,12 @@
struct vm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */
int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
/*
* for testing the idle page zero loop.
*/
boolean_t vm_page_zero_enable = TRUE;
/*
* local variables
*/
@ -217,8 +223,10 @@ uvm_page_init(kvm_startp, kvm_endp)
/*
* step 1: init the page queues and page queue locks
*/
for (lcv = 0; lcv < VM_NFREELIST; lcv++)
TAILQ_INIT(&uvm.page_free[lcv]);
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
for (i = 0; i < PGFL_NQUEUES; i++)
TAILQ_INIT(&uvm.page_free[lcv].pgfl_queues[i]);
}
TAILQ_INIT(&uvm.page_active);
TAILQ_INIT(&uvm.page_inactive_swp);
TAILQ_INIT(&uvm.page_inactive_obj);
@ -333,6 +341,16 @@ uvm_page_init(kvm_startp, kvm_endp)
uvmexp.reserve_pagedaemon = 1;
uvmexp.reserve_kernel = 5;
/*
* step 8: determine if we should zero pages in the idle
* loop.
*
* XXXJRT - might consider zero'ing up to the target *now*,
* but that could take an awfully long time if you
* have a lot of memory.
*/
uvm.page_idle_zero = vm_page_zero_enable;
/*
* done!
*/
@ -848,6 +866,12 @@ uvm_page_physdump()
* => only one of obj or anon can be non-null
* => caller must activate/deactivate page if it is not wired.
* => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
* => policy decision: it is more important to pull a page off of the
* appropriate priority free list than it is to get a zero'd or
* unknown contents page. This is because we live with the
* consequences of a bad free list decision for the entire
* lifetime of the page, e.g. if the page comes from memory that
* is slower to access.
*/
struct vm_page *
@ -858,9 +882,10 @@ uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
struct vm_anon *anon;
int strat, free_list;
{
int lcv, s;
int lcv, try1, try2, s, zeroit = 0;
struct vm_page *pg;
struct pglist *freeq;
struct pgfreelist *pgfl;
boolean_t use_reserve;
#ifdef DIAGNOSTIC
@ -896,13 +921,32 @@ uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
!(use_reserve && curproc == uvm.pagedaemon_proc)))
goto fail;
#if PGFL_NQUEUES != 2
#error uvm_pagealloc_strat needs to be updated
#endif
/*
* If we want a zero'd page, try the ZEROS queue first, otherwise
* we try the UNKNOWN queue first.
*/
if (flags & UVM_PGA_ZERO) {
try1 = PGFL_ZEROS;
try2 = PGFL_UNKNOWN;
} else {
try1 = PGFL_UNKNOWN;
try2 = PGFL_ZEROS;
}
again:
switch (strat) {
case UVM_PGA_STRAT_NORMAL:
/* Check all freelists in descending priority order. */
for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
freeq = &uvm.page_free[lcv];
if ((pg = freeq->tqh_first) != NULL)
pgfl = &uvm.page_free[lcv];
if ((pg = TAILQ_FIRST((freeq =
&pgfl->pgfl_queues[try1]))) != NULL ||
(pg = TAILQ_FIRST((freeq =
&pgfl->pgfl_queues[try2]))) != NULL)
goto gotit;
}
@ -917,8 +961,11 @@ uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
panic("uvm_pagealloc_strat: bad free list %d",
free_list);
#endif
freeq = &uvm.page_free[free_list];
if ((pg = freeq->tqh_first) != NULL)
pgfl = &uvm.page_free[free_list];
if ((pg = TAILQ_FIRST((freeq =
&pgfl->pgfl_queues[try1]))) != NULL ||
(pg = TAILQ_FIRST((freeq =
&pgfl->pgfl_queues[try2]))) != NULL)
goto gotit;
/* Fall back, if possible. */
@ -939,6 +986,24 @@ uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
TAILQ_REMOVE(freeq, pg, pageq);
uvmexp.free--;
/* update zero'd page count */
if (pg->flags & PG_ZERO)
uvmexp.zeropages--;
/*
* update allocation statistics and remember if we have to
* zero the page
*/
if (flags & UVM_PGA_ZERO) {
if (pg->flags & PG_ZERO) {
uvmexp.pga_zerohit++;
zeroit = 0;
} else {
uvmexp.pga_zeromiss++;
zeroit = 1;
}
}
uvm_unlock_fpageq(s); /* unlock free page queue */
pg->offset = off;
@ -963,11 +1028,12 @@ uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
if (flags & UVM_PGA_ZERO) {
/*
* This is an inline of uvm_pagezero(); this will change
* when we have pre-zero'd pages.
* A zero'd page is not clean. If we got a page not already
* zero'd, then we have to zero it ourselves.
*/
pg->flags &= ~PG_CLEAN;
pmap_zero_page(VM_PAGE_TO_PHYS(pg));
if (zeroit)
pmap_zero_page(VM_PAGE_TO_PHYS(pg));
}
return(pg);
@ -1105,8 +1171,7 @@ struct vm_page *pg;
/*
* if the page was wired, unwire it now.
*/
if (pg->wire_count)
{
if (pg->wire_count) {
pg->wire_count = 0;
uvmexp.wired--;
}
@ -1115,9 +1180,11 @@ struct vm_page *pg;
* and put on free queue
*/
pg->flags &= ~PG_ZERO;
s = uvm_lock_fpageq();
TAILQ_INSERT_TAIL(&uvm.page_free[uvm_page_lookup_freelist(pg)],
pg, pageq);
TAILQ_INSERT_TAIL(&uvm.page_free[
uvm_page_lookup_freelist(pg)].pgfl_queues[PGFL_UNKNOWN], pg, pageq);
pg->pqflags = PQ_FREE;
#ifdef DEBUG
pg->uobject = (void *)0xdeadbeef;
@ -1125,6 +1192,10 @@ struct vm_page *pg;
pg->uanon = (void *)0xdeadbeef;
#endif
uvmexp.free++;
if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
uvm.page_idle_zero = vm_page_zero_enable;
uvm_unlock_fpageq(s);
}
@ -1166,3 +1237,66 @@ uvm_page_own(pg, tag)
return;
}
#endif
/*
* uvm_pageidlezero: zero free pages while the system is idle.
*
* => we do at least one iteration per call, if we are below the target.
* => we loop until we either reach the target or whichqs indicates that
* there is a process ready to run.
*/
void
uvm_pageidlezero()
{
struct vm_page *pg;
struct pgfreelist *pgfl;
int free_list, s;
do {
s = uvm_lock_fpageq();
if (uvmexp.zeropages >= UVM_PAGEZERO_TARGET) {
uvm.page_idle_zero = FALSE;
uvm_unlock_fpageq(s);
return;
}
for (free_list = 0; free_list < VM_NFREELIST; free_list++) {
pgfl = &uvm.page_free[free_list];
if ((pg = TAILQ_FIRST(&pgfl->pgfl_queues[
PGFL_UNKNOWN])) != NULL)
break;
}
if (pg == NULL) {
/*
* No non-zero'd pages; don't bother trying again
* until we know we have non-zero'd pages free.
*/
uvm.page_idle_zero = FALSE;
uvm_unlock_fpageq(s);
return;
}
TAILQ_REMOVE(&pgfl->pgfl_queues[PGFL_UNKNOWN], pg, pageq);
uvmexp.free--;
uvm_unlock_fpageq(s);
#ifdef PMAP_PAGEIDLEZERO
PMAP_PAGEIDLEZERO(VM_PAGE_TO_PHYS(pg));
#else
/*
* XXX This will toast the cache unless the pmap_zero_page()
* XXX implementation does uncached access.
*/
pmap_zero_page(VM_PAGE_TO_PHYS(pg));
#endif
pg->flags |= PG_ZERO;
s = uvm_lock_fpageq();
TAILQ_INSERT_HEAD(&pgfl->pgfl_queues[PGFL_ZEROS], pg, pageq);
uvmexp.free++;
uvmexp.zeropages++;
uvm_unlock_fpageq(s);
} while (whichqs == 0);
}

View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_page.h,v 1.14 2000/03/26 20:54:47 kleink Exp $ */
/* $NetBSD: uvm_page.h,v 1.15 2000/04/24 17:12:01 thorpej Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -75,6 +75,12 @@
#ifdef _KERNEL
/*
* globals
*/
extern boolean_t vm_page_zero_enable;
/*
* macros
*/
@ -85,6 +91,8 @@
#define uvm_pagehash(obj,off) \
(((unsigned long)obj+(unsigned long)atop(off)) & uvm.page_hashmask)
#define UVM_PAGEZERO_TARGET (uvmexp.free)
/*
* handle inline options
*/
@ -107,6 +115,7 @@ void uvm_page_own __P((struct vm_page *, char *));
boolean_t uvm_page_physget __P((paddr_t *));
#endif
void uvm_page_rehash __P((void));
void uvm_pageidlezero __P((void));
PAGE_INLINE int uvm_lock_fpageq __P((void));
PAGE_INLINE void uvm_unlock_fpageq __P((int));

View File

@ -1,7 +1,5 @@
/* $NetBSD: uvm_pglist.c,v 1.8 1999/07/22 22:58:39 thorpej Exp $ */
/* $NetBSD: uvm_pglist.c,v 1.9 2000/04/24 17:12:01 thorpej Exp $ */
#define VM_PAGE_ALLOC_MEMORY_STATS
/*-
* Copyright (c) 1997 The NetBSD Foundation, Inc.
* All rights reserved.
@ -52,8 +50,6 @@
#include <sys/proc.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_kern.h>
#include <uvm/uvm.h>
@ -100,7 +96,7 @@ uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
paddr_t try, idxpa, lastidxpa;
int psi;
struct vm_page *pgs;
int s, tryidx, idx, end, error, free_list;
int s, tryidx, idx, pgflidx, end, error, free_list;
vm_page_t m;
u_long pagemask;
#ifdef DEBUG
@ -109,10 +105,10 @@ uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
#ifdef DIAGNOSTIC
if ((alignment & (alignment - 1)) != 0)
panic("vm_page_alloc_memory: alignment must be power of 2");
panic("uvm_pglistalloc: alignment must be power of 2");
if ((boundary & (boundary - 1)) != 0)
panic("vm_page_alloc_memory: boundary must be power of 2");
panic("uvm_pglistalloc: boundary must be power of 2");
#endif
/*
@ -139,10 +135,8 @@ uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
s = uvm_lock_fpageq(); /* lock free page queue */
/* Are there even any free pages? */
for (idx = 0; idx < VM_NFREELIST; idx++)
if (uvm.page_free[idx].tqh_first != NULL)
break;
if (idx == VM_NFREELIST)
if (uvmexp.free <= (uvmexp.reserve_pagedaemon +
uvmexp.reserve_kernel))
goto out;
for (;; try += alignment) {
@ -206,6 +200,10 @@ uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
}
}
#if PGFL_NQUEUES != 2
#error uvm_pglistalloc needs to be updated
#endif
/*
* we have a chunk of memory that conforms to the requested constraints.
*/
@ -213,17 +211,23 @@ uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
while (idx < end) {
m = &pgs[idx];
free_list = uvm_page_lookup_freelist(m);
pgflidx = (m->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN;
#ifdef DEBUG
for (tp = uvm.page_free[free_list].tqh_first;
tp != NULL; tp = tp->pageq.tqe_next) {
for (tp = TAILQ_FIRST(&uvm.page_free[
free_list].pgfl_queues[pgflidx]);
tp != NULL;
tp = TAILQ_NEXT(tp, pageq)) {
if (tp == m)
break;
}
if (tp == NULL)
panic("uvm_pglistalloc: page not on freelist");
#endif
TAILQ_REMOVE(&uvm.page_free[free_list], m, pageq);
TAILQ_REMOVE(&uvm.page_free[free_list].pgfl_queues[pgflidx],
m, pageq);
uvmexp.free--;
if (m->flags & PG_ZERO)
uvmexp.zeropages--;
m->flags = PG_CLEAN;
m->pqflags = 0;
m->uobject = NULL;
@ -278,9 +282,12 @@ uvm_pglistfree(list)
#endif
TAILQ_REMOVE(list, m, pageq);
m->pqflags = PQ_FREE;
TAILQ_INSERT_TAIL(&uvm.page_free[uvm_page_lookup_freelist(m)],
TAILQ_INSERT_TAIL(&uvm.page_free[
uvm_page_lookup_freelist(m)].pgfl_queues[PGFL_UNKNOWN],
m, pageq);
uvmexp.free++;
if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
uvm.page_idle_zero = vm_page_zero_enable;
STAT_DECR(uvm_pglistalloc_npages);
}

View File

@ -1,8 +1,60 @@
/* $NetBSD: pglist.h,v 1.2 1999/02/15 04:17:35 hubertf Exp $ */
/* $NetBSD: pglist.h,v 1.3 2000/04/24 17:12:02 thorpej Exp $ */
/*-
* Copyright (c) 2000 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Jason R. Thorpe.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the NetBSD
* Foundation, Inc. and its contributors.
* 4. Neither the name of The NetBSD Foundation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _PGLIST_H_
#define _PGLIST_H_
/*
* This defines the type of a page queue, e.g. active list, inactive
* list, etc.
*/
TAILQ_HEAD(pglist, vm_page);
/*
* A page free list consists of free pages of unknown contents and free
* pages of all zeros.
*/
#define PGFL_UNKNOWN 0
#define PGFL_ZEROS 1
#define PGFL_NQUEUES 2
struct pgfreelist {
struct pglist pgfl_queues[PGFL_NQUEUES];
};
#endif

View File

@ -1,4 +1,4 @@
/* $NetBSD: vm_page.h,v 1.35 2000/03/26 20:54:48 kleink Exp $ */
/* $NetBSD: vm_page.h,v 1.36 2000/04/24 17:12:02 thorpej Exp $ */
/*
* Copyright (c) 1991, 1993
@ -147,12 +147,17 @@ struct vm_page {
* PQ_ ==> lock by page queue lock
* PQ_FREE is locked by free queue lock and is mutex with all other PQs
*
* PG_ZERO is used to indicate that a page has been pre-zero'd. This flag
* is only set when the page is on no queues, and is cleared when the page
* is placed on the free list.
*
* possible deadwood: PG_FAULTING, PQ_LAUNDRY
*/
#define PG_CLEAN 0x0008 /* page has not been modified */
#define PG_BUSY 0x0010 /* page is in transit */
#define PG_WANTED 0x0020 /* someone is waiting for page */
#define PG_TABLED 0x0040 /* page is in VP table */
#define PG_ZERO 0x0100 /* page is pre-zero'd */
#define PG_FAKE 0x0200 /* page is placeholder for pagein */
#define PG_FILLED 0x0400 /* client flag to set when filled */
#define PG_DIRTY 0x0800 /* client flag to set when dirty */