Aren/NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

revert uvm_fault.c 1.104 for now. see PR/28372, PR/32665.

Browse Source
This commit is contained in:
yamt 2006-01-30 11:50:17 +00:00
parent ed1116428c
commit 7fcddedf0b
1 changed files with 102 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

289
sys/uvm/uvm_fault.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_fault.c,v 1.104 2006/01/21 13:13:07 yamt Exp $ */ /* $NetBSD: uvm_fault.c,v 1.105 2006/01/30 11:50:17 yamt Exp $ */
/* /*
* *
@ -39,7 +39,7 @@
*/ */
#include <sys/cdefs.h> #include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.104 2006/01/21 13:13:07 yamt Exp $"); __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.105 2006/01/30 11:50:17 yamt Exp $");
#include "opt_uvmhist.h" #include "opt_uvmhist.h"
@ -535,143 +535,6 @@ released:
/*NOTREACHED*/ /*NOTREACHED*/
} }
/*
* uvmfault_promote: promote data to a new anon. used for 1B and 2B.
*
* 1. allocate an anon and a page.
* 2. fill its contents.
* 3. put it into amap.
*
* => if we fail (result != 0) we unlock everything.
* => on success, return a new locked anon via 'nanon'.
* (*nanon)->an_page will be a resident, locked, dirty page.
*/
static int
uvmfault_promote(struct uvm_faultinfo *ufi,
struct vm_anon *oanon,
struct vm_page *uobjpage,
struct vm_anon **nanon, /* OUT: allocated anon */
struct vm_anon **spare)
{
struct vm_amap *amap = ufi->entry->aref.ar_amap;
struct uvm_object *uobj;
struct vm_anon *anon;
struct vm_page *pg;
struct vm_page *opg;
int error;
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
if (oanon) {
/* anon COW */
opg = oanon->an_page;
KASSERT(opg != NULL);
KASSERT(opg->uobject == NULL || opg->loan_count > 0);
} else if (uobjpage != PGO_DONTCARE) {
/* object-backed COW */
opg = uobjpage;
KASSERT(opg->uobject == ufi->entry->object.uvm_obj);
} else {
/* ZFOD */
opg = NULL;
}
if (opg != NULL) {
uobj = opg->uobject;
} else {
uobj = NULL;
}
KASSERT(amap != NULL);
KASSERT(uobjpage != NULL);
KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
LOCK_ASSERT(simple_lock_held(&amap->am_l));
LOCK_ASSERT(oanon == NULL || simple_lock_held(&oanon->an_lock));
LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
LOCK_ASSERT(*spare == NULL || !simple_lock_held(&(*spare)->an_lock));
if (*spare != NULL) {
anon = *spare;
*spare = NULL;
simple_lock(&anon->an_lock);
} else if (ufi->map != kernel_map) {
anon = uvm_analloc();
} else {
UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
/*
* we can't allocate anons with kernel_map locked.
*/
uvm_page_unbusy(&uobjpage, 1);
uvmfault_unlockall(ufi, amap, uobj, oanon);
*spare = uvm_analloc();
if (*spare == NULL) {
goto nomem;
}
simple_unlock(&(*spare)->an_lock);
error = ERESTART;
goto done;
}
if (anon) {
/*
* The new anon is locked.
*
* if opg == NULL, we want a zero'd, dirty page,
* so have uvm_pagealloc() do that for us.
*/
pg = uvm_pagealloc(NULL, 0, anon,
(opg == NULL) ? UVM_PGA_ZERO : 0);
} else {
pg = NULL;
}
/*
* out of memory resources?
*/
if (pg == NULL) {
/* save anon for the next try. */
if (anon != NULL) {
simple_unlock(&anon->an_lock);
*spare = anon;
}
/* unlock and fail ... */
uvm_page_unbusy(&uobjpage, 1);
uvmfault_unlockall(ufi, amap, uobj, oanon);
nomem:
if (!uvm_reclaimable()) {
UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
uvmexp.fltnoanon++;
error = ENOMEM;
goto done;
}
UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
uvmexp.fltnoram++;
uvm_wait("flt_noram5");
error = ERESTART;
goto done;
}
/* copy page [pg now dirty] */
if (opg) {
uvm_pagecopy(opg, pg);
}
amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
oanon != NULL);
*nanon = anon;
error = 0;
done:
return error;
}
/* /*
* F A U L T - m a i n e n t r y p o i n t * F A U L T - m a i n e n t r y p o i n t
*/ */
@ -704,14 +567,13 @@ uvm_fault(struct vm_map *orig_map, vaddr_t vaddr, vm_fault_t fault_type,
struct vm_amap *amap; struct vm_amap *amap;
struct uvm_object *uobj; struct uvm_object *uobj;
struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
struct vm_anon *anon_spare;
struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)",
orig_map, vaddr, fault_type, access_type); orig_map, vaddr, fault_type, access_type);
anon = anon_spare = NULL; anon = NULL;
pg = NULL; pg = NULL;
uvmexp.faults++; /* XXX: locking? */ uvmexp.faults++; /* XXX: locking? */
@ -742,8 +604,7 @@ ReFault:
if (uvmfault_lookup(&ufi, FALSE) == FALSE) { if (uvmfault_lookup(&ufi, FALSE) == FALSE) {
UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0);
error = EFAULT; return (EFAULT);
goto done;
} }
/* locked: maps(read) */ /* locked: maps(read) */
@ -768,8 +629,7 @@ ReFault:
"<- protection failure (prot=0x%x, access=0x%x)", "<- protection failure (prot=0x%x, access=0x%x)",
ufi.entry->protection, access_type, 0, 0); ufi.entry->protection, access_type, 0, 0);
uvmfault_unlockmaps(&ufi, FALSE); uvmfault_unlockmaps(&ufi, FALSE);
error = EACCES; return EACCES;
goto done;
} }
/* /*
@ -832,8 +692,7 @@ ReFault:
if (amap == NULL && uobj == NULL) { if (amap == NULL && uobj == NULL) {
uvmfault_unlockmaps(&ufi, FALSE); uvmfault_unlockmaps(&ufi, FALSE);
UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
error = EFAULT; return (EFAULT);
goto done;
} }
/* /*
@ -1027,7 +886,7 @@ ReFault:
/* /*
* object fault routine responsible for pmap_update(). * object fault routine responsible for pmap_update().
*/ */
goto done; return error;
} }
/* /*
@ -1205,7 +1064,7 @@ ReFault:
goto ReFault; goto ReFault;
default: default:
goto done; return error;
} }
/* /*
@ -1321,24 +1180,41 @@ ReFault:
UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
uvmexp.flt_acow++; uvmexp.flt_acow++;
oanon = anon; /* oanon = old, locked anon */ oanon = anon; /* oanon = old, locked anon */
anon = uvm_analloc();
error = uvmfault_promote(&ufi, oanon, PGO_DONTCARE, if (anon) {
&anon, &anon_spare); /* new anon is locked! */
switch (error) { pg = uvm_pagealloc(NULL, 0, anon, 0);
case 0:
break;
case ERESTART:
goto ReFault;
default:
goto done;
} }
pg = anon->an_page; /* check for out of RAM */
if (anon == NULL || pg == NULL) {
if (anon) {
anon->an_ref--;
simple_unlock(&anon->an_lock);
uvm_anfree(anon);
}
uvmfault_unlockall(&ufi, amap, uobj, oanon);
if (!uvm_reclaimable()) {
UVMHIST_LOG(maphist,
"<- failed. out of VM",0,0,0,0);
uvmexp.fltnoanon++;
return ENOMEM;
}
uvmexp.fltnoram++;
uvm_wait("flt_noram3"); /* out of RAM, wait for more */
goto ReFault;
}
/* got all resources, replace anon with nanon */
uvm_pagecopy(oanon->an_page, pg);
uvm_lock_pageq(); uvm_lock_pageq();
uvm_pageactivate(pg); uvm_pageactivate(pg);
uvm_unlock_pageq();
pg->flags &= ~(PG_BUSY|PG_FAKE); pg->flags &= ~(PG_BUSY|PG_FAKE);
uvm_unlock_pageq();
UVM_PAGE_OWN(pg, NULL); UVM_PAGE_OWN(pg, NULL);
amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
anon, TRUE);
/* deref: can not drop to zero here by defn! */ /* deref: can not drop to zero here by defn! */
oanon->an_ref--; oanon->an_ref--;
@ -1386,8 +1262,7 @@ ReFault:
UVMHIST_LOG(maphist, UVMHIST_LOG(maphist,
"<- failed. out of VM",0,0,0,0); "<- failed. out of VM",0,0,0,0);
/* XXX instrumentation */ /* XXX instrumentation */
error = ENOMEM; return ENOMEM;
goto done;
} }
/* XXX instrumentation */ /* XXX instrumentation */
uvm_wait("flt_pmfail1"); uvm_wait("flt_pmfail1");
@ -1424,8 +1299,7 @@ ReFault:
simple_unlock(&anon->an_lock); simple_unlock(&anon->an_lock);
uvmfault_unlockall(&ufi, amap, uobj, oanon); uvmfault_unlockall(&ufi, amap, uobj, oanon);
pmap_update(ufi.orig_map->pmap); pmap_update(ufi.orig_map->pmap);
error = 0; return 0;
goto done;
Case2: Case2:
/* /*
@ -1496,7 +1370,7 @@ Case2:
UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
error, 0,0,0); error, 0,0,0);
goto done; return error;
} }
/* locked: uobjpage */ /* locked: uobjpage */
@ -1648,18 +1522,62 @@ Case2:
if (amap == NULL) if (amap == NULL)
panic("uvm_fault: want to promote data, but no anon"); panic("uvm_fault: want to promote data, but no anon");
#endif #endif
error = uvmfault_promote(&ufi, NULL, uobjpage,
&anon, &anon_spare); anon = uvm_analloc();
switch (error) { if (anon) {
case 0:
break; /*
case ERESTART: * The new anon is locked.
goto ReFault; *
default: * In `Fill in data...' below, if
goto done; * uobjpage == PGO_DONTCARE, we want
* a zero'd, dirty page, so have
* uvm_pagealloc() do that for us.
*/
pg = uvm_pagealloc(NULL, 0, anon,
(uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0);
} }
pg = anon->an_page; /*
* out of memory resources?
*/
if (anon == NULL || pg == NULL) {
if (anon != NULL) {
anon->an_ref--;
simple_unlock(&anon->an_lock);
uvm_anfree(anon);
}
/*
* arg! must unbusy our page and fail or sleep.
*/
if (uobjpage != PGO_DONTCARE) {
if (uobjpage->flags & PG_WANTED)
/* still holding object lock */
wakeup(uobjpage);
uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(uobjpage, NULL);
}
/* unlock and fail ... */
uvmfault_unlockall(&ufi, amap, uobj, NULL);
if (!uvm_reclaimable()) {
UVMHIST_LOG(maphist, " promote: out of VM",
0,0,0,0);
uvmexp.fltnoanon++;
return ENOMEM;
}
UVMHIST_LOG(maphist, " out of RAM, waiting for more",
0,0,0,0);
uvmexp.fltnoram++;
uvm_wait("flt_noram5");
goto ReFault;
}
/* /*
* fill in the data * fill in the data
@ -1667,6 +1585,8 @@ Case2:
if (uobjpage != PGO_DONTCARE) { if (uobjpage != PGO_DONTCARE) {
uvmexp.flt_prcopy++; uvmexp.flt_prcopy++;
/* copy page [pg now dirty] */
uvm_pagecopy(uobjpage, pg);
/* /*
* promote to shared amap? make sure all sharing * promote to shared amap? make sure all sharing
@ -1702,13 +1622,15 @@ Case2:
uvmexp.flt_przero++; uvmexp.flt_przero++;
/* /*
* Page is zero'd and marked dirty by * Page is zero'd and marked dirty by uvm_pagealloc()
* uvmfault_promote(). * above.
*/ */
UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x",
anon, pg, 0, 0); anon, pg, 0, 0);
} }
amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
anon, FALSE);
} }
/* /*
@ -1756,8 +1678,7 @@ Case2:
UVMHIST_LOG(maphist, UVMHIST_LOG(maphist,
"<- failed. out of VM",0,0,0,0); "<- failed. out of VM",0,0,0,0);
/* XXX instrumentation */ /* XXX instrumentation */
error = ENOMEM; return ENOMEM;
goto done;
} }
/* XXX instrumentation */ /* XXX instrumentation */
uvm_wait("flt_pmfail2"); uvm_wait("flt_pmfail2");
@ -1796,13 +1717,7 @@ Case2:
uvmfault_unlockall(&ufi, amap, uobj, anon); uvmfault_unlockall(&ufi, amap, uobj, anon);
pmap_update(ufi.orig_map->pmap); pmap_update(ufi.orig_map->pmap);
UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
error = 0; return 0;
done:
if (anon_spare != NULL) {
anon_spare->an_ref--;
uvm_anfree(anon_spare);
}
return error;
} }
/* /*