Aren
/
NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge from yamt-pagecache (after much testing): 

- Reduce unnecessary page scan in putpages esp. when an object has a ton of
  pages cached but only a few of them are dirty.

- Reduce the number of pmap operations by tracking page dirtiness more
  precisely in uvm layer.
This commit is contained in:
ad 2020-01-15 17:55:43 +00:00
parent 5425c6e737
commit 05a3457e85
34 changed files with 885 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
external/cddl/osnet/dist/uts/common/fs/zfs/zfs_vnops.c vendored
View File

@ -746,7 +746,8 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio)
pp = NULL;
npages = 1;
mutex_enter(mtx);
found = uvn_findpages(uobj, start, &npages, &pp, UFP_NOALLOC);
found = uvn_findpages(uobj, start, &npages, &pp, NULL,
UFP_NOALLOC);
mutex_exit(mtx);
/* XXXNETBSD shouldn't access userspace with the page busy */
@ -792,7 +793,8 @@ update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid,
pp = NULL;
npages = 1;
found = uvn_findpages(uobj, start, &npages, &pp, UFP_NOALLOC);
found = uvn_findpages(uobj, start, &npages, &pp, NULL,
UFP_NOALLOC);
if (found) {
mutex_exit(mtx);
@ -5976,7 +5978,7 @@ zfs_netbsd_getpages(void *v)
}
npages = 1;
pg = NULL;
uvn_findpages(uobj, offset, &npages, &pg, UFP_ALL);
uvn_findpages(uobj, offset, &npages, &pg, NULL, UFP_ALL);
if (pg->flags & PG_FAKE) {
mutex_exit(mtx);
@ -6224,7 +6226,7 @@ zfs_netbsd_setsize(vnode_t *vp, off_t size)
mutex_enter(mtx);
count = 1;
pg = NULL;
if (uvn_findpages(uobj, tsize, &count, &pg, UFP_NOALLOC)) {
if (uvn_findpages(uobj, tsize, &count, &pg, NULL, UFP_NOALLOC)) {
va = zfs_map_page(pg, S_WRITE);
pgoff = size - tsize;
memset(va + pgoff, 0, PAGESIZE - pgoff);

10
sys/external/bsd/drm2/dist/drm/drm_gem.c vendored
View File

@ -1,4 +1,4 @@
/* $NetBSD: drm_gem.c,v 1.10 2018/08/27 15:22:53 riastradh Exp $ */
/* $NetBSD: drm_gem.c,v 1.11 2020/01/15 17:55:43 ad Exp $ */
/*
* Copyright © 2008 Intel Corporation
@ -28,7 +28,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: drm_gem.c,v 1.10 2018/08/27 15:22:53 riastradh Exp $");
__KERNEL_RCSID(0, "$NetBSD: drm_gem.c,v 1.11 2020/01/15 17:55:43 ad Exp $");
#include <linux/types.h>
#include <linux/slab.h>
@ -612,8 +612,10 @@ drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages, bool dirty,
unsigned i;
for (i = 0; i < (obj->size >> PAGE_SHIFT); i++) {
if (dirty)
pages[i]->p_vmp.flags &= ~PG_CLEAN;
if (dirty) {
uvm_pagemarkdirty(&pages[i]->p_vmp,
UVM_PAGE_STATUS_DIRTY);
}
}
uvm_obj_unwirepages(obj->filp, 0, obj->size);

6
sys/external/bsd/drm2/dist/drm/i915/i915_gem.c vendored
View File

@ -1,4 +1,4 @@
/* $NetBSD: i915_gem.c,v 1.54 2018/08/27 15:22:54 riastradh Exp $ */
/* $NetBSD: i915_gem.c,v 1.55 2020/01/15 17:55:43 ad Exp $ */
/*
* Copyright © 2008-2015 Intel Corporation
@ -28,7 +28,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: i915_gem.c,v 1.54 2018/08/27 15:22:54 riastradh Exp $");
__KERNEL_RCSID(0, "$NetBSD: i915_gem.c,v 1.55 2020/01/15 17:55:43 ad Exp $");
#ifdef __NetBSD__
#if 0 /* XXX uvmhist option? */
@ -2644,7 +2644,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
if (obj->dirty) {
TAILQ_FOREACH(page, &obj->pageq, pageq.queue) {
page->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(page, UVM_PAGE_STATUS_DIRTY);
/* XXX mark page accessed */
}
}

6
sys/external/bsd/drm2/dist/drm/i915/i915_gem_fence.c vendored
View File

@ -1,4 +1,4 @@
/* $NetBSD: i915_gem_fence.c,v 1.5 2018/08/27 15:09:35 riastradh Exp $ */
/* $NetBSD: i915_gem_fence.c,v 1.6 2020/01/15 17:55:43 ad Exp $ */
/*
* Copyright © 2008-2015 Intel Corporation
@ -24,7 +24,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: i915_gem_fence.c,v 1.5 2018/08/27 15:09:35 riastradh Exp $");
__KERNEL_RCSID(0, "$NetBSD: i915_gem_fence.c,v 1.6 2020/01/15 17:55:43 ad Exp $");
#include <drm/drmP.h>
#include <drm/i915_drm.h>
@ -769,7 +769,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
(test_bit(i, obj->bit_17) != 0)) {
i915_gem_swizzle_page(container_of(page, struct page,
p_vmp));
page->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(page, UVM_PAGE_STATUS_DIRTY);
}
i += 1;
}

4
sys/external/bsd/drm2/include/linux/mm.h vendored
View File

@ -1,4 +1,4 @@
/* $NetBSD: mm.h,v 1.9 2018/08/27 13:44:54 riastradh Exp $ */
/* $NetBSD: mm.h,v 1.10 2020/01/15 17:55:44 ad Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@ -96,7 +96,7 @@ static inline void
set_page_dirty(struct page *page)
{
page->p_vmp.flags &= ~PG_CLEAN;
uvm_pagemarkdirty(&page->p_vmp, UVM_PAGE_STATUS_DIRTY);
}
#endif /* _LINUX_MM_H_ */

282
sys/miscfs/genfs/genfs_io.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: genfs_io.c,v 1.83 2019/12/31 22:42:50 ad Exp $ */
/* $NetBSD: genfs_io.c,v 1.84 2020/01/15 17:55:44 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.83 2019/12/31 22:42:50 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.84 2020/01/15 17:55:44 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -86,10 +86,8 @@ genfs_rel_pages(struct vm_page **pgs, unsigned int npages)
static void
genfs_markdirty(struct vnode *vp)
{
struct genfs_node * const gp = VTOG(vp);
KASSERT(mutex_owned(vp->v_interlock));
gp->g_dirtygen++;
if ((vp->v_iflag & VI_ONWORKLST) == 0) {
vn_syncer_add_to_worklist(vp, filedelay);
}
@ -137,6 +135,7 @@ genfs_getpages(void *v)
UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx/%jx count %jd",
(uintptr_t)vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count);
KASSERT(memwrite >= overwrite);
KASSERT(vp->v_type == VREG || vp->v_type == VDIR ||
vp->v_type == VLNK || vp->v_type == VBLK);
@ -231,12 +230,17 @@ startover:
}
#endif /* defined(DEBUG) */
nfound = uvn_findpages(uobj, origoffset, &npages,
ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
ap->a_m, NULL,
UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
KASSERT(npages == *ap->a_count);
if (nfound == 0) {
error = EBUSY;
goto out_err;
}
/*
* lock and unlock g_glock to ensure that no one is truncating
* the file behind us.
*/
if (!genfs_node_rdtrylock(vp)) {
genfs_rel_pages(ap->a_m, npages);
@ -258,6 +262,17 @@ startover:
}
error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0);
if (error == 0 && memwrite) {
for (i = 0; i < npages; i++) {
pg = ap->a_m[i];
if (pg == NULL || pg == PGO_DONTCARE) {
continue;
}
if (uvm_pagegetdirty(pg) ==
UVM_PAGE_STATUS_CLEAN) {
uvm_pagemarkdirty(pg,
UVM_PAGE_STATUS_UNKNOWN);
}
}
genfs_markdirty(vp);
}
goto out_err;
@ -351,7 +366,7 @@ startover:
goto startover;
}
if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx],
if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], NULL,
async ? UFP_NOWAIT : UFP_ALL) != orignmempages) {
if (!glocked) {
genfs_node_unlock(vp);
@ -363,27 +378,6 @@ startover:
goto out_err_free;
}
/*
* if the pages are already resident, just return them.
*/
for (i = 0; i < npages; i++) {
struct vm_page *pg = pgs[ridx + i];
if ((pg->flags & PG_FAKE) ||
(blockalloc && (pg->flags & PG_RDONLY))) {
break;
}
}
if (i == npages) {
if (!glocked) {
genfs_node_unlock(vp);
}
UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0);
npages += ridx;
goto out;
}
/*
* if PGO_OVERWRITE is set, don't bother reading the pages.
*/
@ -397,12 +391,49 @@ startover:
for (i = 0; i < npages; i++) {
struct vm_page *pg = pgs[ridx + i];
pg->flags &= ~(PG_RDONLY|PG_CLEAN);
/*
* it's caller's responsibility to allocate blocks
* beforehand for the overwrite case.
*/
KASSERT((pg->flags & PG_RDONLY) == 0 || !blockalloc);
pg->flags &= ~PG_RDONLY;
/*
* mark the page DIRTY.
* otherwise another thread can do putpages and pull
* our vnode from syncer's queue before our caller does
* ubc_release. note that putpages won't see CLEAN
* pages even if they are BUSY.
*/
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
}
npages += ridx;
goto out;
}
/*
* if the pages are already resident, just return them.
*/
for (i = 0; i < npages; i++) {
struct vm_page *pg = pgs[ridx + i];
if ((pg->flags & PG_FAKE) ||
(blockalloc && (pg->flags & PG_RDONLY) != 0)) {
break;
}
}
if (i == npages) {
if (!glocked) {
genfs_node_unlock(vp);
}
UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0);
npages += ridx;
goto out;
}
/*
* the page wasn't resident and we're not overwriting,
* so we're going to have to do some i/o.
@ -425,7 +456,7 @@ startover:
UVMHIST_LOG(ubchist, "reset npages start 0x%jx end 0x%jx",
startoffset, endoffset, 0,0);
npgs = npages;
if (uvn_findpages(uobj, startoffset, &npgs, pgs,
if (uvn_findpages(uobj, startoffset, &npgs, pgs, NULL,
async ? UFP_NOWAIT : UFP_ALL) != npages) {
if (!glocked) {
genfs_node_unlock(vp);
@ -473,8 +504,16 @@ out:
UVMHIST_LOG(ubchist, "examining pg %#jx flags 0x%jx",
(uintptr_t)pg, pg->flags, 0,0);
if (pg->flags & PG_FAKE && !overwrite) {
pg->flags &= ~(PG_FAKE);
pmap_clear_modify(pgs[i]);
/*
* we've read page's contents from the backing storage.
*
* for a read fault, we keep them CLEAN; if we
* encountered a hole while reading, the pages can
* already been dirtied with zeros.
*/
KASSERTMSG(blockalloc || uvm_pagegetdirty(pg) ==
UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg);
pg->flags &= ~PG_FAKE;
}
KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0);
if (i < ridx || i >= ridx + orignmempages || async) {
@ -496,6 +535,13 @@ out:
uvm_pageunlock(pg);
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
} else if (memwrite && !overwrite &&
uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) {
/*
* for a write fault, start dirtiness tracking of
* requested pages.
*/
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN);
}
}
if (memwrite) {
@ -690,16 +736,13 @@ genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages,
iobytes);
skipbytes += iobytes;
mutex_enter(uobj->vmobjlock);
for (i = 0; i < holepages; i++) {
if (memwrite) {
pgs[pidx + i]->flags &= ~PG_CLEAN;
}
if (!blockalloc) {
if (!blockalloc) {
mutex_enter(uobj->vmobjlock);
for (i = 0; i < holepages; i++) {
pgs[pidx + i]->flags |= PG_RDONLY;
}
mutex_exit(uobj->vmobjlock);
}
mutex_exit(uobj->vmobjlock);
continue;
}
@ -764,7 +807,8 @@ loopdone:
if (pg == NULL) {
continue;
}
pg->flags &= ~(PG_CLEAN|PG_RDONLY);
pg->flags &= ~PG_RDONLY;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
UVMHIST_LOG(ubchist, "mark dirty pg %#jx",
(uintptr_t)pg, 0, 0, 0);
}
@ -793,11 +837,11 @@ loopdone:
* this routine is holding the lock on the object. the only time
* that it can run into a PG_BUSY page that it does not own is if
* some other process has started I/O on the page (e.g. either
* a pagein, or a pageout). if the PG_BUSY page is being paged
* in, then it can not be dirty (!PG_CLEAN) because no one has
* had a chance to modify it yet. if the PG_BUSY page is being
* paged out then it means that someone else has already started
* cleaning the page for us (how nice!). in this case, if we
* a pagein, or a pageout). if the PG_BUSY page is being paged
* in, then it can not be dirty (!UVM_PAGE_STATUS_CLEAN) because no
* one has had a chance to modify it yet. if the PG_BUSY page is
* being paged out then it means that someone else has already started
* cleaning the page for us (how nice!). in this case, if we
* have syncio specified, then after we make our pass through the
* object we need to wait for the other PG_BUSY pages to clear
* off (i.e. we need to do an iosync). also note that once a
@ -839,14 +883,13 @@ genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff,
bool async = (origflags & PGO_SYNCIO) == 0;
bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
struct lwp * const l = curlwp ? curlwp : &lwp0;
struct genfs_node * const gp = VTOG(vp);
struct mount *trans_mp;
int flags;
int dirtygen;
bool modified;
bool modified; /* if we write out any pages */
bool holds_wapbl;
bool cleanall;
bool cleanall; /* try to pull off from the syncer's list */
bool onworklst;
const bool dirtyonly = (origflags & (PGO_DEACTIVATE|PGO_FREE)) == 0;
UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist);
@ -870,7 +913,14 @@ retry:
flags = origflags;
KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 ||
(vp->v_iflag & VI_WRMAPDIRTY) == 0);
if (uobj->uo_npages == 0) {
/*
* shortcut if we have no pages to process.
*/
if (uobj->uo_npages == 0 || (dirtyonly &&
radix_tree_empty_tagged_tree_p(&uobj->uo_pages,
UVM_PAGE_DIRTY_TAG))) {
if (vp->v_iflag & VI_ONWORKLST) {
vp->v_iflag &= ~VI_WRMAPDIRTY;
if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
@ -940,7 +990,7 @@ retry:
if ((vp->v_iflag & VI_ONWORKLST) == 0) {
#if !defined(DEBUG)
if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) {
if (dirtyonly) {
goto skip_scan;
}
#endif /* !defined(DEBUG) */
@ -951,18 +1001,23 @@ retry:
* start the loop to scan pages.
*/
cleanall = (flags & PGO_CLEANIT) != 0 && wasclean &&
startoff == 0 && endoff == trunc_page(LLONG_MAX) &&
(vp->v_iflag & VI_ONWORKLST) != 0;
dirtygen = gp->g_dirtygen;
cleanall = true;
freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED;
uvm_page_array_init(&a);
for (;;) {
bool pgprotected;
/*
* if the current page is not interesting, move on to the next.
* if !dirtyonly, iterate over all resident pages in the range.
*
* if dirtyonly, only possibly dirty pages are interesting.
* however, if we are asked to sync for integrity, we should
* wait on pages being written back by other threads as well.
*/
pg = uvm_page_array_fill_and_peek(&a, uobj, nextoff, 0, 0);
pg = uvm_page_array_fill_and_peek(&a, uobj, nextoff, 0,
dirtyonly ? (UVM_PAGE_ARRAY_FILL_DIRTY |
(!async ? UVM_PAGE_ARRAY_FILL_WRITEBACK : 0)) : 0);
if (pg == NULL) {
break;
}
@ -972,18 +1027,15 @@ retry:
(pg->flags & (PG_BUSY)) != 0);
KASSERT(pg->offset >= startoff);
KASSERT(pg->offset >= nextoff);
KASSERT(!dirtyonly ||
uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN ||
radix_tree_get_tag(&uobj->uo_pages,
pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG));
if (pg->offset >= endoff) {
break;
}
if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
wasclean = false;
nextoff = pg->offset + PAGE_SIZE;
uvm_page_array_advance(&a);
continue;
}
/*
* a preempt point.
*/
@ -1003,14 +1055,14 @@ retry:
}
/*
* if the current page needs to be cleaned and it's busy,
* wait for it to become unbusy.
* if the current page is busy, wait for it to become unbusy.
*/
if (pg->flags & PG_BUSY) {
if ((pg->flags & PG_BUSY) != 0) {
UVMHIST_LOG(ubchist, "busy %#jx", (uintptr_t)pg,
0, 0, 0);
if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) {
if ((pg->flags & (PG_RELEASED|PG_PAGEOUT)) != 0
&& (flags & PGO_BUSYFAIL) != 0) {
UVMHIST_LOG(ubchist, "busyfail %#jx",
(uintptr_t)pg, 0, 0, 0);
error = EDEADLK;
@ -1025,6 +1077,16 @@ retry:
*/
break;
}
/*
* don't bother to wait on other's activities
* unless we are asked to sync for integrity.
*/
if (!async && (flags & PGO_RECLAIM) == 0) {
wasclean = false;
nextoff = pg->offset + PAGE_SIZE;
uvm_page_array_advance(&a);
continue;
}
nextoff = pg->offset; /* visit this page again */
pg->flags |= PG_WANTED;
UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
@ -1045,8 +1107,10 @@ retry:
* if we're cleaning, check if the page is needs to be cleaned.
*/
pgprotected = false;
if (flags & PGO_FREE) {
pmap_page_protect(pg, VM_PROT_NONE);
pgprotected = true;
} else if (flags & PGO_CLEANIT) {
/*
@ -1054,8 +1118,7 @@ retry:
* from the syncer queue, write-protect the page.
*/
if (cleanall && wasclean &&
gp->g_dirtygen == dirtygen) {
if (cleanall && wasclean) {
/*
* uobj pages get wired only by uvm_fault
@ -1065,6 +1128,7 @@ retry:
if (pg->wire_count == 0) {
pmap_page_protect(pg,
VM_PROT_READ|VM_PROT_EXECUTE);
pgprotected = true;
} else {
cleanall = false;
}
@ -1072,17 +1136,14 @@ retry:
}
if (flags & PGO_CLEANIT) {
needs_clean = pmap_clear_modify(pg) ||
(pg->flags & PG_CLEAN) == 0;
pg->flags |= PG_CLEAN;
needs_clean = uvm_pagecheckdirty(pg, pgprotected);
} else {
needs_clean = false;
}
/*
* if we're cleaning, build a cluster.
* the cluster will consist of pages which are currently dirty,
* but they will be returned to us marked clean.
* the cluster will consist of pages which are currently dirty.
* if not cleaning, just operate on the one page.
*/
@ -1118,7 +1179,8 @@ retry:
npages = (off - lo) >> PAGE_SHIFT;
nback = npages;
uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0],
uvn_findpages(uobj, off - PAGE_SIZE, &nback,
&pgs[0], NULL,
UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD);
if (nback) {
memmove(&pgs[0], &pgs[npages - nback],
@ -1140,6 +1202,14 @@ retry:
/*
* then look forward to fill in the remaining space in
* the array of pages.
*
* pass our cached array of pages so that hopefully
* uvn_findpages can find some good pages in it.
* the array a was filled above with the one of
* following sets of flags:
* 0
* UVM_PAGE_ARRAY_FILL_DIRTY
* UVM_PAGE_ARRAY_FILL_DIRTY|WRITEBACK
*/
npages = MAXPAGES - nback - 1;
@ -1147,7 +1217,7 @@ retry:
npages = MIN(npages,
(fshi - off - 1) >> PAGE_SHIFT);
uvn_findpages(uobj, off + PAGE_SIZE, &npages,
&pgs[nback + 1],
&pgs[nback + 1], NULL,
UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY);
npages += nback + 1;
} else {
@ -1163,6 +1233,19 @@ retry:
for (i = 0; i < npages; i++) {
tpg = pgs[i];
KASSERT(tpg->uobject == uobj);
KASSERT(i == 0 ||
pgs[i-1]->offset + PAGE_SIZE == tpg->offset);
KASSERT(!needs_clean || uvm_pagegetdirty(pgs[i]) !=
UVM_PAGE_STATUS_DIRTY);
if (needs_clean) {
/*
* mark pages as WRITEBACK so that concurrent
* fsync can find and wait for our activities.
*/
radix_tree_set_tag(&uobj->uo_pages,
pgs[i]->offset >> PAGE_SHIFT,
UVM_PAGE_WRITEBACK_TAG);
}
if (tpg->offset < startoff || tpg->offset >= endoff)
continue;
if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
@ -1224,6 +1307,16 @@ retry:
}
uvm_page_array_fini(&a);
/*
* update ctime/mtime if the modification we started writing out might
* be from mmap'ed write.
*
* this is necessary when an application keeps a file mmaped and
* repeatedly modifies it via the window. note that, because we
* don't always write-protect pages when cleaning, such modifications
* might not involve any page faults.
*/
if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 &&
(vp->v_type != VBLK ||
(vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
@ -1231,34 +1324,13 @@ retry:
}
/*
* if we're cleaning and there was nothing to clean,
* take us off the syncer list. if we started any i/o
* and we're doing sync i/o, wait for all writes to finish.
* if we no longer have any possibly dirty pages, take us off the
* syncer list.
*/
if (cleanall && wasclean && gp->g_dirtygen == dirtygen &&
(vp->v_iflag & VI_ONWORKLST) != 0) {
#if defined(DEBUG)
uvm_page_array_init(&a);
for (nextoff = 0;; nextoff = pg->offset + PAGE_SIZE) {
pg = uvm_page_array_fill_and_peek(&a, uobj, nextoff,
0, 0);
if (pg == NULL) {
break;
}
uvm_page_array_advance(&a);
if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) {
continue;
}
if ((pg->flags & PG_CLEAN) == 0) {
printf("%s: %p: !CLEAN\n", __func__, pg);
}
if (pmap_is_modified(pg)) {
printf("%s: %p: modified\n", __func__, pg);
}
}
uvm_page_array_fini(&a);
#endif /* defined(DEBUG) */
if ((vp->v_iflag & VI_ONWORKLST) != 0 &&
radix_tree_empty_tagged_tree_p(&uobj->uo_pages,
UVM_PAGE_DIRTY_TAG)) {
vp->v_iflag &= ~VI_WRMAPDIRTY;
if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
vn_syncer_remove_from_worklist(vp);
@ -1557,7 +1629,7 @@ genfs_compat_getpages(void *v)
pgs = ap->a_m;
if (ap->a_flags & PGO_LOCKED) {
uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m,
uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, NULL,
UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0));
error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0;
@ -1575,7 +1647,7 @@ genfs_compat_getpages(void *v)
return 0;
}
npages = orignpages;
uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL);
uvn_findpages(uobj, origoffset, &npages, pgs, NULL, UFP_ALL);
mutex_exit(uobj->vmobjlock);
kva = uvm_pagermapin(pgs, npages,
UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
@ -1608,7 +1680,7 @@ genfs_compat_getpages(void *v)
if (error && (pg->flags & PG_FAKE) != 0) {
pg->flags |= PG_RELEASED;
} else {
pmap_clear_modify(pg);
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN);
uvm_pagelock(pg);
uvm_pageactivate(pg);
uvm_pageunlock(pg);

3
sys/miscfs/genfs/genfs_node.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: genfs_node.h,v 1.22 2018/05/28 21:04:38 chs Exp $ */
/* $NetBSD: genfs_node.h,v 1.23 2020/01/15 17:55:44 ad Exp $ */
/*
* Copyright (c) 2001 Chuck Silvers.
@ -80,7 +80,6 @@ struct genfs_ops {
struct genfs_node {
const struct genfs_ops *g_op; /* ops vector */
krwlock_t g_glock; /* getpages lock */
int g_dirtygen;
};
#define VTOG(vp) ((struct genfs_node *)(vp)->v_data)

7
sys/nfs/nfs_bio.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: nfs_bio.c,v 1.192 2019/12/13 20:10:21 ad Exp $ */
/* $NetBSD: nfs_bio.c,v 1.193 2020/01/15 17:55:44 ad Exp $ */
/*
* Copyright (c) 1989, 1993
@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.192 2019/12/13 20:10:21 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: nfs_bio.c,v 1.193 2020/01/15 17:55:44 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_nfs.h"
@ -1120,7 +1120,8 @@ again:
*/
mutex_enter(uobj->vmobjlock);
for (i = 0; i < npages; i++) {
pgs[i]->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pgs[i],
UVM_PAGE_STATUS_DIRTY);
}
mutex_exit(uobj->vmobjlock);
}

4
sys/rump/librump/rumpkern/Makefile.rumpkern
View File

@ -1,4 +1,4 @@
# $NetBSD: Makefile.rumpkern,v 1.181 2019/12/20 21:20:09 ad Exp $
# $NetBSD: Makefile.rumpkern,v 1.182 2020/01/15 17:55:44 ad Exp $
#
IOCONFDIR:= ${.PARSEDIR}
@ -139,7 +139,7 @@ SRCS+= init_sysctl_base.c \
# sys/uvm
SRCS+= uvm_aobj.c uvm_readahead.c uvm_object.c uvm_swapstub.c
SRCS+= uvm_page_array.c
SRCS+= uvm_page_array.c uvm_page_status.c
# 4.4BSD secmodel. selection is hardcoded for now
SRCS+= secmodel.c

6
sys/rump/librump/rumpkern/vm.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: vm.c,v 1.182 2020/01/05 15:57:15 para Exp $ */
/* $NetBSD: vm.c,v 1.183 2020/01/15 17:55:44 ad Exp $ */
/*
* Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved.
@ -41,7 +41,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.182 2020/01/05 15:57:15 para Exp $");
__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.183 2020/01/15 17:55:44 ad Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@ -235,7 +235,7 @@ void
uvm_pagezero(struct vm_page *pg)
{
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
memset((void *)pg->uanon, 0, PAGE_SIZE);
}

12
sys/rump/librump/rumpvfs/vm_vfs.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: vm_vfs.c,v 1.35 2019/12/13 20:10:22 ad Exp $ */
/* $NetBSD: vm_vfs.c,v 1.36 2020/01/15 17:55:44 ad Exp $ */
/*
* Copyright (c) 2008-2011 Antti Kantee. All Rights Reserved.
@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: vm_vfs.c,v 1.35 2019/12/13 20:10:22 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: vm_vfs.c,v 1.36 2020/01/15 17:55:44 ad Exp $");
#include <sys/param.h>
@ -141,7 +141,7 @@ ubc_zerorange(struct uvm_object *uobj, off_t off, size_t len, int flags)
start = (uint8_t *)pg->uanon + chunkoff;
memset(start, 0, chunklen);
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
off += chunklen;
len -= chunklen;
@ -210,8 +210,10 @@ ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo,
mutex_exit(uobj->vmobjlock);
goto out;
}
if (uio->uio_rw == UIO_WRITE)
pg->flags &= ~(PG_CLEAN | PG_FAKE);
if (uio->uio_rw == UIO_WRITE) {
pg->flags &= ~PG_FAKE;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
}
todo -= xfersize;
}
uvm_page_unbusy(pgs, npages);

12
sys/sys/cpu_data.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: cpu_data.h,v 1.48 2020/01/12 13:29:24 ad Exp $ */
/* $NetBSD: cpu_data.h,v 1.49 2020/01/15 17:55:44 ad Exp $ */
/*-
* Copyright (c) 2004, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@ -87,7 +87,15 @@ enum cpu_count {
CPU_COUNT_FLTNOANON,
CPU_COUNT_FLTNORAM,
CPU_COUNT_FLTPGRELE,
CPU_COUNT_MAX /* 40 */
CPU_COUNT_ANONUNKNOWN, /* 40 */
CPU_COUNT_ANONCLEAN,
CPU_COUNT_ANONDIRTY,
CPU_COUNT_FILEUNKNOWN,
CPU_COUNT_FILECLEAN,
CPU_COUNT_FILEDIRTY,
CPU_COUNT__UNUSED1,
CPU_COUNT__UNUSED2,
CPU_COUNT_MAX /* 48 */
};
/*

15
sys/ufs/lfs/lfs_pages.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: lfs_pages.c,v 1.19 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: lfs_pages.c,v 1.20 2020/01/15 17:55:44 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc.
@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.19 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.20 2020/01/15 17:55:44 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_compat_netbsd.h"
@ -306,8 +306,10 @@ check_dirty(struct lfs *fs, struct vnode *vp,
UVM_PAGE_OWN(pg, "lfs_putpages");
pmap_page_protect(pg, VM_PROT_NONE);
tdirty = (pmap_clear_modify(pg) ||
(pg->flags & PG_CLEAN) == 0);
tdirty =
uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN &&
(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY ||
pmap_clear_modify(pg));
dirty += tdirty;
}
if ((pages_per_block > 0 && nonexistent >= pages_per_block) ||
@ -329,10 +331,11 @@ check_dirty(struct lfs *fs, struct vnode *vp,
for (i = 0; i == 0 || i < pages_per_block; i++) {
KASSERT(mutex_owned(vp->v_interlock));
pg = pgs[i];
KASSERT(!((pg->flags & PG_CLEAN) && (pg->flags & PG_DELWRI)));
KASSERT(!(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_DIRTY
&& (pg->flags & PG_DELWRI)));
KASSERT(pg->flags & PG_BUSY);
if (dirty) {
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
if (flags & PGO_FREE) {
/*
* Wire the page so that

7
sys/ufs/lfs/lfs_segment.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: lfs_segment.c,v 1.280 2019/12/08 19:52:37 ad Exp $ */
/* $NetBSD: lfs_segment.c,v 1.281 2020/01/15 17:55:44 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc.
@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.280 2019/12/08 19:52:37 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.281 2020/01/15 17:55:44 ad Exp $");
#ifdef DEBUG
# define vndebug(vp, str) do { \
@ -241,7 +241,8 @@ lfs_vflush(struct vnode *vp)
pg = uvm_pagelookup(&vp->v_uobj, off);
if (pg == NULL)
continue;
if ((pg->flags & PG_CLEAN) == 0 ||
if (uvm_pagegetdirty(pg)
== UVM_PAGE_STATUS_DIRTY ||
pmap_is_modified(pg)) {
lfs_sb_addavail(fs,
lfs_btofsb(fs,

7
sys/ufs/lfs/lfs_vfsops.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: lfs_vfsops.c,v 1.367 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: lfs_vfsops.c,v 1.368 2020/01/15 17:55:44 ad Exp $ */
/*-
* Copyright (c) 1999, 2000, 2001, 2002, 2003, 2007, 2007
@ -61,7 +61,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: lfs_vfsops.c,v 1.367 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: lfs_vfsops.c,v 1.368 2020/01/15 17:55:44 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_lfs.h"
@ -2249,7 +2249,8 @@ lfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages,
}
uvm_pageactivate(pg);
uvm_pageunlock(pg);
pg->flags &= ~(PG_CLEAN|PG_DELWRI|PG_PAGEOUT|PG_RELEASED);
pg->flags &= ~(PG_DELWRI|PG_PAGEOUT|PG_RELEASED);
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
DLOG((DLOG_PAGE, "pg[%d] = %p (vp %p off %" PRIx64 ")\n", i, pg,
vp, pg->offset));
DLOG((DLOG_PAGE, "pg[%d]->flags = %x\n", i, pg->flags));

20
sys/ufs/lfs/ulfs_inode.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: ulfs_inode.c,v 1.23 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: ulfs_inode.c,v 1.24 2020/01/15 17:55:44 ad Exp $ */
/* from NetBSD: ufs_inode.c,v 1.95 2015/06/13 14:56:45 hannken Exp */
/*
@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ulfs_inode.c,v 1.23 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: ulfs_inode.c,v 1.24 2020/01/15 17:55:44 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_lfs.h"
@ -223,13 +223,13 @@ ulfs_balloc_range(struct vnode *vp, off_t off, off_t len, kauth_cred_t cred,
genfs_node_unlock(vp);
/*
* if the allocation succeeded, clear PG_CLEAN on all the pages
* and clear PG_RDONLY on any pages that are now fully backed
* by disk blocks. if the allocation failed, we do not invalidate
* the pages since they might have already existed and been dirty,
* in which case we need to keep them around. if we created the pages,
* they will be clean and read-only, and leaving such pages
* in the cache won't cause any problems.
* if the allocation succeeded, mark all pages dirty and clear
* PG_RDONLY on any pages that are now fully backed by disk blocks.
* if the allocation failed, we do not invalidate the pages since
* they might have already existed and been dirty, in which case we
* need to keep them around. if we created the pages, they will be
* clean and read-only, and leaving such pages in the cache won't
* cause any problems.
*/
GOP_SIZE(vp, off + len, &eob, 0);
@ -241,7 +241,7 @@ ulfs_balloc_range(struct vnode *vp, off_t off, off_t len, kauth_cred_t cred,
pagestart + ((i + 1) << PAGE_SHIFT) <= eob) {
pgs[i]->flags &= ~PG_RDONLY;
}
pgs[i]->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pgs[i], UVM_PAGE_STATUS_DIRTY);
}
uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);

8
sys/ufs/ufs/ufs_inode.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: ufs_inode.c,v 1.107 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: ufs_inode.c,v 1.108 2020/01/15 17:55:44 ad Exp $ */
/*
* Copyright (c) 1991, 1993
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ufs_inode.c,v 1.107 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: ufs_inode.c,v 1.108 2020/01/15 17:55:44 ad Exp $");
#if defined(_KERNEL_OPT)
#include "opt_ffs.h"
@ -259,7 +259,7 @@ ufs_balloc_range(struct vnode *vp, off_t off, off_t len, kauth_cred_t cred,
genfs_node_unlock(vp);
/*
* if the allocation succeeded, clear PG_CLEAN on all the pages
* if the allocation succeeded, mark all the pages dirty
* and clear PG_RDONLY on any pages that are now fully backed
* by disk blocks. if the allocation failed, we do not invalidate
* the pages since they might have already existed and been dirty,
@ -277,7 +277,7 @@ ufs_balloc_range(struct vnode *vp, off_t off, off_t len, kauth_cred_t cred,
pagestart + ((i + 1) << PAGE_SHIFT) <= eob) {
pgs[i]->flags &= ~PG_RDONLY;
}
pgs[i]->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pgs[i], UVM_PAGE_STATUS_DIRTY);
}
uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);

3
sys/uvm/files.uvm
View File

@ -1,4 +1,4 @@
# $NetBSD: files.uvm,v 1.32 2019/12/27 12:51:57 ad Exp $
# $NetBSD: files.uvm,v 1.33 2020/01/15 17:55:45 ad Exp $
#
# UVM options
@ -38,6 +38,7 @@ file uvm/uvm_mremap.c uvm
file uvm/uvm_object.c uvm
file uvm/uvm_page.c uvm
file uvm/uvm_page_array.c uvm
file uvm/uvm_page_status.c uvm
file uvm/uvm_pager.c uvm
file uvm/uvm_pdaemon.c uvm
file uvm/uvm_pdpolicy_clock.c !pdpolicy_clockpro

6
sys/uvm/uvm_anon.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_anon.c,v 1.70 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_anon.c,v 1.71 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.70 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.71 2020/01/15 17:55:45 ad Exp $");
#include "opt_uvmhist.h"
@ -346,7 +346,7 @@ uvm_anon_pagein(struct vm_amap *amap, struct vm_anon *anon)
uvm_swap_free(anon->an_swslot, 1);
}
anon->an_swslot = 0;
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
/*
* Deactivate the page (to put it on a page queue).

28
sys/uvm/uvm_aobj.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_aobj.c,v 1.133 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_aobj.c,v 1.134 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@ -38,7 +38,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.133 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.134 2020/01/15 17:55:45 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@ -849,7 +849,8 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
if (ptmp) {
/* new page */
ptmp->flags &= ~(PG_FAKE);
ptmp->flags |= PG_AOBJ;
uvm_pagemarkdirty(ptmp,
UVM_PAGE_STATUS_UNKNOWN);
goto gotpage;
}
}
@ -870,6 +871,8 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
* useful page: busy/lock it and plug it in our
* result array
*/
KASSERT(uvm_pagegetdirty(ptmp) !=
UVM_PAGE_STATUS_CLEAN);
/* caller must un-busy this page */
ptmp->flags |= PG_BUSY;
@ -951,8 +954,6 @@ gotpage:
continue;
}
ptmp->flags |= PG_AOBJ;
/*
* got new page ready for I/O. break pps while
* loop. pps[lcv] is still NULL.
@ -980,6 +981,8 @@ gotpage:
* loop).
*/
KASSERT(uvm_pagegetdirty(ptmp) !=
UVM_PAGE_STATUS_CLEAN);
/* we own it, caller must un-busy */
ptmp->flags |= PG_BUSY;
UVM_PAGE_OWN(ptmp, "uao_get2");
@ -1060,10 +1063,11 @@ gotpage:
#endif /* defined(VMSWAP) */
}
if ((access_type & VM_PROT_WRITE) == 0) {
ptmp->flags |= PG_CLEAN;
pmap_clear_modify(ptmp);
}
/*
* note that we will allow the page being writably-mapped
* (!PG_RDONLY) regardless of access_type.
*/
uvm_pagemarkdirty(ptmp, UVM_PAGE_STATUS_UNKNOWN);
/*
* we got the page! clear the fake flag (indicates valid
@ -1075,7 +1079,8 @@ gotpage:
* => unbusy the page
* => activate the page
*/
KASSERT(uvm_pagegetdirty(ptmp) != UVM_PAGE_STATUS_CLEAN);
KASSERT((ptmp->flags & PG_FAKE) != 0);
ptmp->flags &= ~PG_FAKE;
pps[lcv] = ptmp;
}
@ -1308,7 +1313,8 @@ uao_pagein_page(struct uvm_aobj *aobj, int pageidx)
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_CLEAN|PG_FAKE);
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
UVM_PAGE_OWN(pg, NULL);
return false;

35
sys/uvm/uvm_bio.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_bio.c,v 1.102 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_bio.c,v 1.103 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1998 Chuck Silvers.
@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.102 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.103 2020/01/15 17:55:45 ad Exp $");
#include "opt_uvmhist.h"
#include "opt_ubc.h"
@ -230,13 +230,11 @@ static inline int
ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va)
{
struct uvm_object *uobj;
vm_prot_t mask;
int error;
bool rdonly;
uobj = pg->uobject;
KASSERT(mutex_owned(uobj->vmobjlock));
KASSERT(mutex_owned(pg->uobject->vmobjlock));
if (pg->flags & PG_WANTED) {
wakeup(pg);
@ -270,6 +268,9 @@ ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
/*
* Note that a page whose backing store is partially allocated
* is marked as PG_RDONLY.
*
* it's a responsibility of ubc_alloc's caller to allocate backing
* blocks before writing to the window.
*/
KASSERT((pg->flags & PG_RDONLY) == 0 ||
@ -277,9 +278,7 @@ ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap,
pg->offset < umap->writeoff ||
pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen);
rdonly = ((access_type & VM_PROT_WRITE) == 0 &&
(pg->flags & PG_RDONLY) != 0) ||
UVM_OBJ_NEEDS_WRITEFAULT(uobj);
rdonly = uvm_pagereadonly_p(pg);
mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL;
error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg),
@ -665,7 +664,10 @@ ubc_release(void *va, int flags)
umapva + slot_offset + (i << PAGE_SHIFT), &pa);
KASSERT(rv);
pgs[i] = PHYS_TO_VM_PAGE(pa);
pgs[i]->flags &= ~(PG_FAKE|PG_CLEAN);
pgs[i]->flags &= ~PG_FAKE;
KASSERTMSG(uvm_pagegetdirty(pgs[i]) ==
UVM_PAGE_STATUS_DIRTY,
"page %p not dirty", pgs[i]);
KASSERT(pgs[i]->loan_count == 0);
uvm_pagelock(pgs[i]);
uvm_pageactivate(pgs[i]);
@ -896,9 +898,18 @@ ubc_direct_release(struct uvm_object *uobj,
uvm_pageactivate(pg);
uvm_pageunlock(pg);
/* Page was changed, no longer fake and neither clean */
if (flags & UBC_WRITE)
pg->flags &= ~(PG_FAKE|PG_CLEAN);
/*
* Page was changed, no longer fake and neither clean.
* There's no managed mapping in the direct case, so
* mark the page dirty manually.
*/
if (flags & UBC_WRITE) {
pg->flags &= ~PG_FAKE;
KASSERTMSG(uvm_pagegetdirty(pg) ==
UVM_PAGE_STATUS_DIRTY,
"page %p not dirty", pg);
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
}
}
uvm_page_unbusy(pgs, npages);
mutex_exit(uobj->vmobjlock);

12
sys/uvm/uvm_extern.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_extern.h,v 1.218 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_extern.h,v 1.219 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -500,6 +500,12 @@ struct uvmexp_sysctl {
int64_t poolpages;
int64_t countsyncone;
int64_t countsyncall;
int64_t anonunknown;
int64_t anonclean;
int64_t anondirty;
int64_t fileunknown;
int64_t fileclean;
int64_t filedirty;
};
#ifdef _KERNEL
@ -779,10 +785,12 @@ int uvm_grow(struct proc *, vaddr_t);
void uvm_deallocate(struct vm_map *, vaddr_t, vsize_t);
/* uvm_vnode.c */
struct uvm_page_array;
void uvm_vnp_setsize(struct vnode *, voff_t);
void uvm_vnp_setwritesize(struct vnode *, voff_t);
int uvn_findpages(struct uvm_object *, voff_t,
int *, struct vm_page **, int);
unsigned int *, struct vm_page **,
struct uvm_page_array *, unsigned int);
bool uvn_text_p(struct uvm_object *);
bool uvn_clean_p(struct uvm_object *);
bool uvn_needs_writefault_p(struct uvm_object *);

67
sys/uvm/uvm_fault.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_fault.c,v 1.214 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_fault.c,v 1.215 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.214 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.215 2020/01/15 17:55:45 ad Exp $");
#include "opt_uvmhist.h"
@ -378,7 +378,7 @@ uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
uvmfault_unlockall(ufi, amap, NULL);
/*
* Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into
* Pass a PG_BUSY+PG_FAKE clean page into
* the uvm_swap_get() function with all data
* structures unlocked. Note that it is OK
* to read an_swslot here, because we hold
@ -488,6 +488,7 @@ released:
uvm_pageactivate(pg);
uvm_pageunlock(pg);
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN);
UVM_PAGE_OWN(pg, NULL);
#else
panic("%s: we_own", __func__);
@ -640,6 +641,7 @@ uvmfault_promote(struct uvm_faultinfo *ufi,
if (opg) {
uvm_pagecopy(opg, pg);
}
KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
oanon != NULL);
@ -782,7 +784,7 @@ static inline void uvm_fault_lower_lookup(
struct vm_page **);
static inline void uvm_fault_lower_neighbor(
struct uvm_faultinfo *, const struct uvm_faultctx *,
vaddr_t, struct vm_page *, bool);
vaddr_t, struct vm_page *);
static inline int uvm_fault_lower_io(
struct uvm_faultinfo *, const struct uvm_faultctx *,
struct uvm_object **, struct vm_page **);
@ -1256,6 +1258,11 @@ uvm_fault_upper_neighbor(
/* locked: amap, anon */
KASSERT(pg->uobject == NULL);
KASSERT(pg->uanon != NULL);
KASSERT(mutex_owned(pg->uanon->an_lock));
KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
uvm_pagelock(pg);
uvm_pageenqueue(pg);
uvm_pageunlock(pg);
@ -1535,6 +1542,7 @@ uvm_fault_upper_enter(
KASSERT(anon->an_lock == amap->am_lock);
KASSERT(oanon->an_lock == amap->am_lock);
KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
/*
* now map the page in.
@ -1612,21 +1620,20 @@ uvm_fault_upper_done(
uvm_pagelock(pg);
if (wire_paging) {
uvm_pagewire(pg);
/*
* since the now-wired page cannot be paged out,
* release its swap resources for others to use.
* since an anon with no swap cannot be PG_CLEAN,
* clear its clean flag now.
*/
pg->flags &= ~(PG_CLEAN);
} else {
uvm_pageactivate(pg);
}
uvm_pageunlock(pg);
if (wire_paging) {
/*
* since the now-wired page cannot be paged out,
* release its swap resources for others to use.
* and since an anon with no swap cannot be clean,
* mark it dirty now.
*/
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
uvm_anon_dropswap(anon);
}
}
@ -1744,7 +1751,7 @@ uvm_fault_lower(
KASSERT(uobjpage != NULL);
KASSERT(uobj == NULL || uobj == uobjpage->uobject);
KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
(uobjpage->flags & PG_CLEAN) != 0);
uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN);
if (!flt->promote) {
error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
@ -1813,12 +1820,7 @@ uvm_fault_lower_lookup(
UVMHIST_LOG(maphist, " got uobjpage (0x%#jx) "
"with locked get", (uintptr_t)curpg, 0, 0, 0);
} else {
bool readonly = (curpg->flags & PG_RDONLY)
|| (curpg->loan_count > 0)
|| UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
uvm_fault_lower_neighbor(ufi, flt,
currva, curpg, readonly);
uvm_fault_lower_neighbor(ufi, flt, currva, curpg);
}
}
pmap_update(ufi->orig_map->pmap);
@ -1831,8 +1833,9 @@ uvm_fault_lower_lookup(
static void
uvm_fault_lower_neighbor(
struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
vaddr_t currva, struct vm_page *pg, bool readonly)
vaddr_t currva, struct vm_page *pg)
{
const bool readonly = uvm_pagereadonly_p(pg) || pg->loan_count > 0;
UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
/* locked: maps(read), amap(if there), uobj */
@ -1861,7 +1864,8 @@ uvm_fault_lower_neighbor(
KASSERT((pg->flags & PG_PAGEOUT) == 0);
KASSERT((pg->flags & PG_RELEASED) == 0);
KASSERT((pg->flags & PG_WANTED) == 0);
KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) || (pg->flags & PG_CLEAN) != 0);
KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
pg->flags &= ~(PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
@ -2216,6 +2220,7 @@ uvm_fault_lower_enter(
struct vm_anon *anon, struct vm_page *pg)
{
struct vm_amap * const amap = ufi->entry->aref.ar_amap;
const bool readonly = uvm_pagereadonly_p(pg);
int error;
UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist);
@ -2241,12 +2246,16 @@ uvm_fault_lower_enter(
" MAPPING: case2: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd",
(uintptr_t)ufi->orig_map->pmap, ufi->orig_rvaddr,
(uintptr_t)pg, flt->promote);
KASSERT((flt->access_type & VM_PROT_WRITE) == 0 ||
(pg->flags & PG_RDONLY) == 0);
KASSERTMSG((flt->access_type & VM_PROT_WRITE) == 0 || !readonly,
"promote=%u cow_now=%u access_type=%x enter_prot=%x cow=%u "
"entry=%p map=%p orig_rvaddr=%p pg=%p",
flt->promote, flt->cow_now, flt->access_type, flt->enter_prot,
UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map,
(void *)ufi->orig_rvaddr, pg);
KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly);
if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
VM_PAGE_TO_PHYS(pg),
(pg->flags & PG_RDONLY) != 0 ?
flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
flt->access_type | PMAP_CANFAIL |
(flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
@ -2332,12 +2341,12 @@ uvm_fault_lower_done(
/*
* since the now-wired page cannot be paged out,
* release its swap resources for others to use.
* since an aobj page with no swap cannot be PG_CLEAN,
* clear its clean flag now.
* since an aobj page with no swap cannot be clean,
* mark it dirty now.
*/
KASSERT(uobj != NULL);
pg->flags &= ~(PG_CLEAN);
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
dropswap = true;
}
} else {

35
sys/uvm/uvm_loan.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_loan.c,v 1.93 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_loan.c,v 1.94 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.93 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.94 2020/01/15 17:55:45 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -1126,22 +1126,13 @@ uvm_loanbreak(struct vm_page *uobjpage)
* one and clear the fake flags on the new page (keep it busy).
* force a reload of the old page by clearing it from all
* pmaps.
* transfer dirtiness of the old page to the new page.
* then rename the pages.
*/
uvm_pagecopy(uobjpage, pg); /* old -> new */
pg->flags &= ~PG_FAKE;
KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
pmap_page_protect(uobjpage, VM_PROT_NONE);
if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
pmap_clear_modify(pg);
pg->flags |= PG_CLEAN;
} else {
/* uvm_pagecopy marked it dirty */
KASSERT((pg->flags & PG_CLEAN) == 0);
/* a object with a dirty page should be dirty. */
KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
}
if (uobjpage->flags & PG_WANTED)
wakeup(uobjpage);
/* uobj still locked */
@ -1184,9 +1175,11 @@ int
uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
{
struct vm_page *newpg, *oldpg;
unsigned oldstatus;
KASSERT(mutex_owned(anon->an_lock));
KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
KASSERT(anon->an_page->loan_count > 0);
/* get new un-owned replacement page */
newpg = uvm_pagealloc(NULL, 0, NULL, 0);
@ -1197,24 +1190,29 @@ uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
oldpg = anon->an_page;
/* copy old -> new */
uvm_pagecopy(oldpg, newpg);
KASSERT(uvm_pagegetdirty(newpg) == UVM_PAGE_STATUS_DIRTY);
/* force reload */
pmap_page_protect(oldpg, VM_PROT_NONE);
oldstatus = uvm_pagegetdirty(anon->an_page);
uvm_pagelock2(oldpg, newpg);
if (uobj == NULL) {
/*
* we were the lender (A->K); need to remove the page from
* pageq's.
*
* PG_ANON is updated by the caller.
*/
KASSERT((oldpg->flags & PG_ANON) != 0);
oldpg->flags &= ~PG_ANON;
uvm_pagedequeue(oldpg);
}
oldpg->uanon = NULL;
/* in case we owned */
oldpg->flags &= ~PG_ANON;
if (uobj) {
/* if we were receiver of loan */
KASSERT((oldpg->pqflags & PG_ANON) == 0);
oldpg->loan_count--;
}
@ -1234,6 +1232,13 @@ uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
}
/* done! */
kpreempt_disable();
if (uobj != NULL) {
CPU_COUNT(CPU_COUNT_ANONPAGES, 1);
} else {
CPU_COUNT(CPU_COUNT_ANONUNKNOWN + oldstatus, -1);
}
CPU_COUNT(CPU_COUNT_ANONDIRTY, 1);
kpreempt_enable();
return 0;
}

10
sys/uvm/uvm_meter.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_meter.c,v 1.73 2019/12/31 13:07:14 ad Exp $ */
/* $NetBSD: uvm_meter.c,v 1.74 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_meter.c,v 1.73 2019/12/31 13:07:14 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_meter.c,v 1.74 2020/01/15 17:55:45 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -180,6 +180,12 @@ sysctl_vm_uvmexp2(SYSCTLFN_ARGS)
u.poolpages = pool_totalpages();
u.countsyncone = cpu_count_get(CPU_COUNT_SYNC_ONE);
u.countsyncall = cpu_count_get(CPU_COUNT_SYNC_ALL);
u.anonunknown = (int)cpu_count_get(CPU_COUNT_ANONUNKNOWN);
u.anonclean = (int)cpu_count_get(CPU_COUNT_ANONCLEAN);
u.anondirty = (int)cpu_count_get(CPU_COUNT_ANONDIRTY);
u.fileunknown = (int)cpu_count_get(CPU_COUNT_FILEUNKNOWN);
u.fileclean = (int)cpu_count_get(CPU_COUNT_FILECLEAN);
u.filedirty = (int)cpu_count_get(CPU_COUNT_FILEDIRTY);
node = *rnode;
node.sysctl_data = &u;

7
sys/uvm/uvm_object.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_object.c,v 1.19 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_object.c,v 1.20 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 2006, 2010, 2019 The NetBSD Foundation, Inc.
@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.19 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.20 2020/01/15 17:55:45 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_ddb.h"
@ -174,7 +174,8 @@ uvm_obj_wirepages(struct uvm_object *uobj, off_t start, off_t end,
}
if (pgs[i]->flags & PG_AOBJ) {
pgs[i]->flags &= ~(PG_CLEAN);
uvm_pagemarkdirty(pgs[i],
UVM_PAGE_STATUS_DIRTY);
uao_dropswap(uobj, i);
}
}

9
sys/uvm/uvm_object.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_object.h,v 1.35 2019/12/15 21:11:35 ad Exp $ */
/* $NetBSD: uvm_object.h,v 1.36 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -62,6 +62,13 @@ struct uvm_object {
LIST_HEAD(,ubc_map) uo_ubc; /* ubc mappings */
};
/*
* tags for uo_pages
*/
#define UVM_PAGE_DIRTY_TAG 1 /* might be dirty (!PG_CLEAN) */
#define UVM_PAGE_WRITEBACK_TAG 2 /* being written back */
/*
* UVM_OBJ_KERN is a 'special' uo_refs value which indicates that the
* object is a kernel memory object rather than a normal one (kernel

156
sys/uvm/uvm_page.c
View File

@ -1,7 +1,7 @@
/* $NetBSD: uvm_page.c,v 1.223 2020/01/11 19:51:01 ad Exp $ */
/* $NetBSD: uvm_page.c,v 1.224 2020/01/15 17:55:45 ad Exp $ */
/*-
* Copyright (c) 2019 The NetBSD Foundation, Inc.
* Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@ -95,7 +95,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.223 2020/01/11 19:51:01 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.224 2020/01/15 17:55:45 ad Exp $");
#include "opt_ddb.h"
#include "opt_uvm.h"
@ -217,19 +217,31 @@ uvm_pageinsert_object(struct uvm_object *uobj, struct vm_page *pg)
KASSERT(mutex_owned(uobj->vmobjlock));
KASSERT((pg->flags & PG_TABLED) == 0);
if (UVM_OBJ_IS_VNODE(uobj)) {
if (uobj->uo_npages == 0) {
struct vnode *vp = (struct vnode *)uobj;
if ((pg->flags & PG_STAT) != 0) {
/* Cannot use uvm_pagegetdirty(): not yet in radix tree. */
const unsigned int status = pg->flags & (PG_CLEAN | PG_DIRTY);
const bool isaobj = (pg->flags & PG_AOBJ) != 0;
vholdl(vp);
}
if (UVM_OBJ_IS_VTEXT(uobj)) {
cpu_count(CPU_COUNT_EXECPAGES, 1);
if (!isaobj) {
KASSERT((pg->flags & PG_FILE) != 0);
if (uobj->uo_npages == 0) {
struct vnode *vp = (struct vnode *)uobj;
vholdl(vp);
}
kpreempt_disable();
if (UVM_OBJ_IS_VTEXT(uobj)) {
CPU_COUNT(CPU_COUNT_EXECPAGES, 1);
} else {
CPU_COUNT(CPU_COUNT_FILEPAGES, 1);
}
CPU_COUNT(CPU_COUNT_FILEUNKNOWN + status, 1);
} else {
cpu_count(CPU_COUNT_FILEPAGES, 1);
kpreempt_disable();
CPU_COUNT(CPU_COUNT_ANONPAGES, 1);
CPU_COUNT(CPU_COUNT_ANONUNKNOWN + status, 1);
}
} else if (UVM_OBJ_IS_AOBJ(uobj)) {
cpu_count(CPU_COUNT_ANONPAGES, 1);
kpreempt_enable();
}
pg->flags |= PG_TABLED;
uobj->uo_npages++;
@ -245,6 +257,11 @@ uvm_pageinsert_tree(struct uvm_object *uobj, struct vm_page *pg)
if (error != 0) {
return error;
}
if ((pg->flags & PG_CLEAN) == 0) {
radix_tree_set_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG);
}
KASSERT(((pg->flags & PG_CLEAN) == 0) ==
radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
return 0;
}
@ -262,22 +279,32 @@ uvm_pageremove_object(struct uvm_object *uobj, struct vm_page *pg)
KASSERT(mutex_owned(uobj->vmobjlock));
KASSERT(pg->flags & PG_TABLED);
if (UVM_OBJ_IS_VNODE(uobj)) {
if (uobj->uo_npages == 1) {
struct vnode *vp = (struct vnode *)uobj;
if ((pg->flags & PG_STAT) != 0) {
/* Cannot use uvm_pagegetdirty(): no longer in radix tree. */
const unsigned int status = pg->flags & (PG_CLEAN | PG_DIRTY);
const bool isaobj = (pg->flags & PG_AOBJ) != 0;
holdrelel(vp);
}
if (UVM_OBJ_IS_VTEXT(uobj)) {
cpu_count(CPU_COUNT_EXECPAGES, -1);
if (!isaobj) {
KASSERT((pg->flags & PG_FILE) != 0);
if (uobj->uo_npages == 1) {
struct vnode *vp = (struct vnode *)uobj;
holdrelel(vp);
}
kpreempt_disable();
if (UVM_OBJ_IS_VTEXT(uobj)) {
CPU_COUNT(CPU_COUNT_EXECPAGES, -1);
} else {
CPU_COUNT(CPU_COUNT_FILEPAGES, -1);
}
CPU_COUNT(CPU_COUNT_FILEUNKNOWN + status, -1);
} else {
cpu_count(CPU_COUNT_FILEPAGES, -1);
kpreempt_disable();
CPU_COUNT(CPU_COUNT_ANONPAGES, -1);
CPU_COUNT(CPU_COUNT_ANONUNKNOWN + status, -1);
}
} else if (UVM_OBJ_IS_AOBJ(uobj)) {
cpu_count(CPU_COUNT_ANONPAGES, -1);
kpreempt_enable();
}
/* object should be locked */
uobj->uo_npages--;
pg->flags &= ~PG_TABLED;
pg->uobject = NULL;
@ -1290,6 +1317,7 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
}
if (anon) {
CPU_COUNT(CPU_COUNT_ANONPAGES, 1);
CPU_COUNT(CPU_COUNT_ANONCLEAN, 1);
}
splx(s);
KASSERT((pg->flags & ~(PG_ZERO|PG_FREE)) == 0);
@ -1312,6 +1340,14 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
pg->flags |= PG_ANON;
mutex_exit(&pg->interlock);
} else if (obj) {
/*
* set PG_FILE|PG_AOBJ before the first uvm_pageinsert.
*/
if (UVM_OBJ_IS_VNODE(obj)) {
pg->flags |= PG_FILE;
} else {
pg->flags |= PG_AOBJ;
}
uvm_pageinsert_object(obj, pg);
mutex_exit(&pg->interlock);
error = uvm_pageinsert_tree(obj, pg);
@ -1334,9 +1370,12 @@ uvm_pagealloc_strat(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
* A zero'd page is not clean. If we got a page not already
* zero'd, then we have to zero it ourselves.
*/
pg->flags &= ~PG_CLEAN;
if (zeroit)
if (obj != NULL || anon != NULL) {
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
}
if (zeroit) {
pmap_zero_page(VM_PAGE_TO_PHYS(pg));
}
}
return(pg);
@ -1354,6 +1393,7 @@ uvm_pagereplace(struct vm_page *oldpg, struct vm_page *newpg)
{
struct uvm_object *uobj = oldpg->uobject;
struct vm_page *pg __diagused;
uint64_t idx;
KASSERT((oldpg->flags & PG_TABLED) != 0);
KASSERT(uobj != NULL);
@ -1363,12 +1403,25 @@ uvm_pagereplace(struct vm_page *oldpg, struct vm_page *newpg)
KASSERT(mutex_owned(&oldpg->interlock));
KASSERT(mutex_owned(&newpg->interlock));
newpg->offset = oldpg->offset;
pg = radix_tree_replace_node(&uobj->uo_pages,
newpg->offset >> PAGE_SHIFT, newpg);
KASSERT(pg == oldpg);
newpg->uobject = uobj;
newpg->offset = oldpg->offset;
idx = newpg->offset >> PAGE_SHIFT;
pg = radix_tree_replace_node(&uobj->uo_pages, idx, newpg);
KASSERT(pg == oldpg);
if (((oldpg->flags ^ newpg->flags) & PG_CLEAN) != 0) {
if ((newpg->flags & PG_CLEAN) != 0) {
radix_tree_clear_tag(&uobj->uo_pages, idx,
UVM_PAGE_DIRTY_TAG);
} else {
radix_tree_set_tag(&uobj->uo_pages, idx,
UVM_PAGE_DIRTY_TAG);
}
}
/*
* oldpg's PG_STAT is stable. newpg is not reachable by others yet.
*/
newpg->flags |=
(newpg->flags & ~PG_STAT) | (oldpg->flags & PG_STAT);
uvm_pageinsert_object(uobj, newpg);
uvm_pageremove_object(uobj, oldpg);
}
@ -1502,7 +1555,7 @@ uvm_pagefree(struct vm_page *pg)
locked = true;
if (pg->uobject != NULL) {
uvm_pageremove_object(pg->uobject, pg);
pg->flags &= ~PG_CLEAN;
pg->flags &= ~(PG_FILE|PG_AOBJ);
} else if (pg->uanon != NULL) {
if ((pg->flags & PG_ANON) == 0) {
pg->loan_count--;
@ -1520,6 +1573,7 @@ uvm_pagefree(struct vm_page *pg)
#ifdef UVM_PAGE_TRKOWN
pg->owner_tag = NULL;
#endif
KASSERT((pg->flags & PG_STAT) == 0);
if (pg->loan_count) {
KASSERT(pg->uobject == NULL);
if (pg->uanon == NULL) {
@ -1542,9 +1596,13 @@ uvm_pagefree(struct vm_page *pg)
if (pg->uobject != NULL) {
uvm_pageremove_object(pg->uobject, pg);
} else if (pg->uanon != NULL) {
const unsigned int status = uvm_pagegetdirty(pg);
pg->uanon->an_page = NULL;
pg->uanon = NULL;
cpu_count(CPU_COUNT_ANONPAGES, -1);
kpreempt_disable();
CPU_COUNT(CPU_COUNT_ANONPAGES, -1);
CPU_COUNT(CPU_COUNT_ANONUNKNOWN + status, -1);
kpreempt_enable();
}
/*
@ -1953,7 +2011,8 @@ uvm_pageunlock2(struct vm_page *pg1, struct vm_page *pg2)
void
uvm_pagezero(struct vm_page *pg)
{
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
pmap_zero_page(VM_PAGE_TO_PHYS(pg));
}
@ -1968,7 +2027,7 @@ void
uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
{
dst->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(dst, UVM_PAGE_STATUS_DIRTY);
pmap_copy_page(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst));
}
@ -2015,6 +2074,29 @@ uvm_page_owner_locked_p(struct vm_page *pg)
return true;
}
/*
* uvm_pagereadonly_p: return if the page should be mapped read-only
*/
bool
uvm_pagereadonly_p(struct vm_page *pg)
{
struct uvm_object * const uobj = pg->uobject;
KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
KASSERT(uobj != NULL || mutex_owned(pg->uanon->an_lock));
if ((pg->flags & PG_RDONLY) != 0) {
return true;
}
if (uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) {
return true;
}
if (uobj == NULL) {
return false;
}
return UVM_OBJ_NEEDS_WRITEFAULT(uobj);
}
#ifdef PMAP_DIRECT
/*
* Call pmap to translate physical address into a virtual and to run a callback
@ -2080,7 +2162,7 @@ uvm_page_printit(struct vm_page *pg, bool full,
(*pr)("PAGE %p:\n", pg);
snprintb(pgbuf, sizeof(pgbuf), page_flagbits, pg->flags);
(*pr)(" flags=%s, pqflags=%x, wire_count=%d, pa=0x%lx\n",
(*pr)(" flags=%s\n pqflags=%x, wire_count=%d, pa=0x%lx\n",
pgbuf, pg->pqflags, pg->wire_count, (long)VM_PAGE_TO_PHYS(pg));
(*pr)(" uobject=%p, uanon=%p, offset=0x%llx loan_count=%d\n",
pg->uobject, pg->uanon, (long long)pg->offset, pg->loan_count);

105
sys/uvm/uvm_page.h
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_page.h,v 1.95 2020/01/10 21:32:17 ad Exp $ */
/* $NetBSD: uvm_page.h,v 1.96 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -160,8 +160,7 @@ struct vm_page {
TAILQ_ENTRY(vm_page) pdqueue; /* p: pagedaemon queue */
kmutex_t interlock; /* s: lock on identity */
uint32_t pqflags; /* i: pagedaemon flags */
uint16_t flags; /* o: object flags */
uint16_t spare; /* : spare for now */
uint32_t flags; /* o: object flags */
paddr_t phys_addr; /* o: physical address of pg */
uint32_t loan_count; /* o,i: num. active loans */
uint32_t wire_count; /* o,i: wired down map refs */
@ -194,6 +193,15 @@ struct vm_page {
*
* Flag descriptions:
*
* PG_CLEAN:
* Page is known clean.
* The contents of the page is consistent with its backing store.
*
* PG_DIRTY:
* Page is known dirty.
* To avoid losing data, the contents of the page should be written
* back to the backing store before freeing the page.
*
* PG_BUSY:
* Page is long-term locked, usually because of I/O (transfer from the
* page memory to the backing store) is in progress. LWP attempting
@ -205,31 +213,20 @@ struct vm_page {
* responsible to clear both flags and wake up any waiters once it has
* released the long-term lock (PG_BUSY).
*
* PG_PAGEOUT:
* Indicates that the page is being paged-out in preparation for
* being freed.
*
* PG_RELEASED:
* Indicates that the page, which is currently PG_BUSY, should be freed
* after the release of long-term lock. It is responsibility of the
* owning LWP (i.e. which set PG_BUSY) to do it.
*
* PG_CLEAN:
* Page has not been modified since it was loaded from the backing
* store. If this flag is not set, page is considered "dirty".
* XXX: Currently it means that the page *might* be clean; will be
* fixed with yamt-pagecache merge.
*
* PG_FAKE:
* Page has been allocated, but not yet initialised. The flag is used
* to avoid overwriting of valid data, e.g. to prevent read from the
* backing store when in-core data is newer.
*
* PG_TABLED:
* Indicates that the page is currently in the object's offset queue,
* and that it should be removed from it once the page is freed. Used
* diagnostic purposes.
*
* PG_PAGEOUT:
* Indicates that the page is being paged-out in preparation for
* being freed.
*
* PG_RDONLY:
* Indicates that the page must be mapped read-only.
*
@ -239,31 +236,43 @@ struct vm_page {
* page is placed on the free list.
*
* PG_MARKER:
* Dummy marker page.
* Dummy marker page, generally used for list traversal.
*/
#define PG_BUSY 0x0001
#define PG_WANTED 0x0002
#define PG_TABLED 0x0004
#define PG_CLEAN 0x0008
#define PG_PAGEOUT 0x0010
#define PG_RELEASED 0x0020
#define PG_FAKE 0x0040
#define PG_RDONLY 0x0080
#define PG_AOBJ 0x0100 /* page is part of an anonymous
/*
* if you want to renumber PG_CLEAN and PG_DIRTY, check __CTASSERTs in
* uvm_page_status.c first.
*/
#define PG_CLEAN 0x00000001 /* page is known clean */
#define PG_DIRTY 0x00000002 /* page is known dirty */
#define PG_BUSY 0x00000004 /* page is locked */
#define PG_WANTED 0x00000008 /* someone is waiting for page */
#define PG_PAGEOUT 0x00000010 /* page to be freed for pagedaemon */
#define PG_RELEASED 0x00000020 /* page to be freed when unbusied */
#define PG_FAKE 0x00000040 /* page is not yet initialized */
#define PG_RDONLY 0x00000080 /* page must be mapped read-only */
#define PG_ZERO 0x00000100 /* page is pre-zero'd */
#define PG_TABLED 0x00000200 /* page is tabled in object */
#define PG_AOBJ 0x00000400 /* page is part of an anonymous
uvm_object */
#define PG_ANON 0x0200 /* page is part of an anon, rather
#define PG_ANON 0x00000800 /* page is part of an anon, rather
than an uvm_object */
#define PG_SWAPBACKED (PG_ANON|PG_AOBJ)
#define PG_READAHEAD 0x0400 /* read-ahead but not "hit" yet */
#define PG_FREE 0x0800 /* page is on free list */
#define PG_MARKER 0x1000
#define PG_PAGER1 0x2000 /* pager-specific flag */
#define PG_ZERO 0x4000
#define PG_FILE 0x00001000 /* file backed (non-anonymous) */
#define PG_READAHEAD 0x00002000 /* read-ahead but not "hit" yet */
#define PG_FREE 0x00004000 /* page is on free list */
#define PG_MARKER 0x00008000 /* dummy marker page */
#define PG_PAGER1 0x00010000 /* pager-specific flag */
#define PG_STAT (PG_ANON|PG_AOBJ|PG_FILE)
#define PG_SWAPBACKED (PG_ANON|PG_AOBJ)
#define UVM_PGFLAGBITS \
"\20\1BUSY\2WANTED\3TABLED\4CLEAN\5PAGEOUT\6RELEASED\7FAKE\10RDONLY" \
"\11AOBJ\12AOBJ\13READAHEAD\14FREE\15MARKER\16PAGER1\17ZERO"
"\20\1CLEAN\2DIRTY\3BUSY\4WANTED" \
"\5PAGEOUT\6RELEASED\7FAKE\10RDONLY" \
"\11ZERO\12TABLED\13AOBJ\14ANON" \
"\15FILE\16READAHEAD\17FREE\20MARKER" \
"\21PAGER1"
/*
* uvmpdpol state flags.
@ -343,6 +352,11 @@ bool uvm_pageismanaged(paddr_t);
bool uvm_page_owner_locked_p(struct vm_page *);
void uvm_pgfl_lock(void);
void uvm_pgfl_unlock(void);
unsigned int uvm_pagegetdirty(struct vm_page *);
void uvm_pagemarkdirty(struct vm_page *, unsigned int);
bool uvm_pagecheckdirty(struct vm_page *, bool);
bool uvm_pagereadonly_p(struct vm_page *);
bool uvm_page_locked_p(struct vm_page *);
int uvm_page_lookup_freelist(struct vm_page *);
@ -355,6 +369,23 @@ int uvm_direct_process(struct vm_page **, u_int, voff_t, vsize_t,
int (*)(void *, size_t, void *), void *);
#endif
/*
* page dirtiness status for uvm_pagegetdirty and uvm_pagemarkdirty
*
* UNKNOWN means that we need to consult pmap to know if the page is
* dirty or not.
* basically, UVM_PAGE_STATUS_CLEAN implies that the page has no writable
* mapping.
*
* if you want to renumber these, check __CTASSERTs in
* uvm_page_status.c first.
*/
#define UVM_PAGE_STATUS_UNKNOWN 0
#define UVM_PAGE_STATUS_CLEAN 1
#define UVM_PAGE_STATUS_DIRTY 2
#define UVM_PAGE_NUM_STATUS 3
/*
* macros
*/

9
sys/uvm/uvm_page_array.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_page_array.c,v 1.2 2019/12/15 21:11:35 ad Exp $ */
/* $NetBSD: uvm_page_array.c,v 1.3 2020/01/15 17:55:45 ad Exp $ */
/*-
* Copyright (c)2011 YAMAMOTO Takashi,
@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_page_array.c,v 1.2 2019/12/15 21:11:35 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_page_array.c,v 1.3 2020/01/15 17:55:45 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@ -142,7 +142,6 @@ uvm_page_array_fill(struct uvm_page_array *ar, struct uvm_object *uobj,
KASSERT(mutex_owned(uobj->vmobjlock));
#endif
KASSERT(uvm_page_array_peek(ar) == NULL);
#if 0 /* not merged from yamt-pagecache yet */
if ((flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0) {
unsigned int tagmask = UVM_PAGE_DIRTY_TAG;
@ -154,9 +153,7 @@ uvm_page_array_fill(struct uvm_page_array *ar, struct uvm_object *uobj,
radix_tree_gang_lookup_tagged_node)(
&uobj->uo_pages, off >> PAGE_SHIFT, (void **)ar->ar_pages,
maxpages, dense, tagmask);
} else
#endif
{
} else {
npages =
(backward ? radix_tree_gang_lookup_node_reverse :
radix_tree_gang_lookup_node)(

194
sys/uvm/uvm_page_status.c Normal file
View File

@ -0,0 +1,194 @@
/* $NetBSD: uvm_page_status.c,v 1.2 2020/01/15 17:55:45 ad Exp $ */
/*-
* Copyright (c)2011 YAMAMOTO Takashi,
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.2 2020/01/15 17:55:45 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
#include <uvm/uvm.h>
/*
* page dirtiness status tracking
*
* separated from uvm_page.c mainly for rump
*/
/*
* these constants are chosen to match so that we can convert between
* them quickly.
*/
__CTASSERT(UVM_PAGE_STATUS_UNKNOWN == 0);
__CTASSERT(UVM_PAGE_STATUS_DIRTY == PG_DIRTY);
__CTASSERT(UVM_PAGE_STATUS_CLEAN == PG_CLEAN);
/*
* uvm_pagegetdirty: return the dirtiness status (one of UVM_PAGE_STATUS_
* values) of the page.
*
* called with the owner locked.
*/
unsigned int
uvm_pagegetdirty(struct vm_page *pg)
{
struct uvm_object * const uobj __diagused = pg->uobject;
const uint64_t idx __diagused = pg->offset >> PAGE_SHIFT;
KASSERT((~pg->flags & (PG_CLEAN|PG_DIRTY)) != 0);
KASSERT(uvm_page_owner_locked_p(pg));
KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
!!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
return pg->flags & (PG_CLEAN|PG_DIRTY);
}
/*
* uvm_pagemarkdirty: set the dirtiness status (one of UVM_PAGE_STATUS_ values)
* of the page.
*
* called with the owner locked.
*
* update the radix tree tag for object-owned page.
*
* if new status is UVM_PAGE_STATUS_UNKNOWN, clear pmap-level dirty bit
* so that later uvm_pagecheckdirty() can notice modifications on the page.
*/
void
uvm_pagemarkdirty(struct vm_page *pg, unsigned int newstatus)
{
struct uvm_object * const uobj = pg->uobject;
const uint64_t idx = pg->offset >> PAGE_SHIFT;
const unsigned int oldstatus = uvm_pagegetdirty(pg);
enum cpu_count base;
KASSERT((~newstatus & (PG_CLEAN|PG_DIRTY)) != 0);
KASSERT((newstatus & ~(PG_CLEAN|PG_DIRTY)) == 0);
KASSERT(uvm_page_owner_locked_p(pg));
KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
!!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
if (oldstatus == newstatus) {
return;
}
/*
* set UVM_PAGE_DIRTY_TAG tag unless known CLEAN so that putpages can
* find possibly-dirty pages quickly.
*/
if (uobj != NULL) {
if (newstatus == UVM_PAGE_STATUS_CLEAN) {
radix_tree_clear_tag(&uobj->uo_pages, idx,
UVM_PAGE_DIRTY_TAG);
} else {
radix_tree_set_tag(&uobj->uo_pages, idx,
UVM_PAGE_DIRTY_TAG);
}
}
if (newstatus == UVM_PAGE_STATUS_UNKNOWN) {
/*
* start relying on pmap-level dirtiness tracking.
*/
pmap_clear_modify(pg);
}
pg->flags &= ~(PG_CLEAN|PG_DIRTY);
pg->flags |= newstatus;
KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
!!radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
if ((pg->flags & PG_STAT) != 0) {
if ((pg->flags & PG_SWAPBACKED) != 0) {
base = CPU_COUNT_ANONUNKNOWN;
} else {
base = CPU_COUNT_FILEUNKNOWN;
}
kpreempt_disable();
CPU_COUNT(base + oldstatus, -1);
CPU_COUNT(base + newstatus, +1);
kpreempt_enable();
}
}
/*
* uvm_pagecheckdirty: check if page is dirty, and remove its dirty bit.
*
* called with the owner locked.
*
* returns if the page was dirty.
*
* if protected is true, mark the page CLEAN. otherwise, mark the page UNKNOWN.
* ("mark" in the sense of uvm_pagemarkdirty().)
*/
bool
uvm_pagecheckdirty(struct vm_page *pg, bool pgprotected)
{
const unsigned int oldstatus = uvm_pagegetdirty(pg);
bool modified;
KASSERT(uvm_page_owner_locked_p(pg));
/*
* if pgprotected is true, mark the page CLEAN.
* otherwise mark the page UNKNOWN unless it's CLEAN.
*
* possible transitions:
*
* CLEAN -> CLEAN , modified = false
* UNKNOWN -> UNKNOWN, modified = true
* UNKNOWN -> UNKNOWN, modified = false
* UNKNOWN -> CLEAN , modified = true
* UNKNOWN -> CLEAN , modified = false
* DIRTY -> UNKNOWN, modified = true
* DIRTY -> CLEAN , modified = true
*
* pmap_clear_modify is necessary if either of
* oldstatus or newstatus is UVM_PAGE_STATUS_UNKNOWN.
*/
if (oldstatus == UVM_PAGE_STATUS_CLEAN) {
modified = false;
} else {
const unsigned int newstatus = pgprotected ?
UVM_PAGE_STATUS_CLEAN : UVM_PAGE_STATUS_UNKNOWN;
if (oldstatus == UVM_PAGE_STATUS_DIRTY) {
modified = true;
if (newstatus == UVM_PAGE_STATUS_UNKNOWN) {
pmap_clear_modify(pg);
}
} else {
KASSERT(oldstatus == UVM_PAGE_STATUS_UNKNOWN);
modified = pmap_clear_modify(pg);
}
uvm_pagemarkdirty(pg, newstatus);
}
return modified;
}

18
sys/uvm/uvm_pager.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_pager.c,v 1.119 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_pager.c,v 1.120 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.119 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.120 2020/01/15 17:55:45 ad Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@ -367,6 +367,13 @@ uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
}
#endif /* defined(VMSWAP) */
if (write && uobj != NULL) {
KASSERT(radix_tree_get_tag(&uobj->uo_pages,
pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG));
radix_tree_clear_tag(&uobj->uo_pages,
pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG);
}
/*
* process errors. for reads, just mark the page to be freed.
* for writes, if the error was ENOMEM, we assume this was
@ -386,7 +393,7 @@ uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
pg->flags &= ~PG_PAGEOUT;
pageout_done++;
}
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
uvm_pagelock(pg);
uvm_pageactivate(pg);
uvm_pageunlock(pg);
@ -413,8 +420,6 @@ uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
/*
* if the page is PG_FAKE, this must have been a read to
* initialize the page. clear PG_FAKE and activate the page.
* we must also clear the pmap "modified" flag since it may
* still be set from the page's previous identity.
*/
if (pg->flags & PG_FAKE) {
@ -424,11 +429,10 @@ uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
pg->flags |= PG_READAHEAD;
uvm_ra_total.ev_count++;
#endif /* defined(READAHEAD_STATS) */
KASSERT((pg->flags & PG_CLEAN) != 0);
KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
uvm_pagelock(pg);
uvm_pageenqueue(pg);
uvm_pageunlock(pg);
pmap_clear_modify(pg);
}
/*

18
sys/uvm/uvm_pdaemon.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_pdaemon.c,v 1.122 2019/12/31 22:42:51 ad Exp $ */
/* $NetBSD: uvm_pdaemon.c,v 1.123 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -66,7 +66,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.122 2019/12/31 22:42:51 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.123 2020/01/15 17:55:45 ad Exp $");
#include "opt_uvmhist.h"
#include "opt_readahead.h"
@ -609,14 +609,14 @@ uvmpd_dropswap(struct vm_page *pg)
if ((pg->flags & PG_ANON) && anon->an_swslot) {
uvm_swap_free(anon->an_swslot, 1);
anon->an_swslot = 0;
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
result = true;
} else if (pg->flags & PG_AOBJ) {
int slot = uao_set_swslot(pg->uobject,
pg->offset >> PAGE_SHIFT, 0);
if (slot) {
uvm_swap_free(slot, 1);
pg->flags &= ~PG_CLEAN;
uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
result = true;
}
}
@ -757,10 +757,14 @@ uvmpd_scan_queue(void)
*/
pmap_page_protect(p, VM_PROT_NONE);
if ((p->flags & PG_CLEAN) && pmap_clear_modify(p)) {
p->flags &= ~(PG_CLEAN);
if (uvm_pagegetdirty(p) == UVM_PAGE_STATUS_UNKNOWN) {
if (pmap_clear_modify(p)) {
uvm_pagemarkdirty(p, UVM_PAGE_STATUS_DIRTY);
} else {
uvm_pagemarkdirty(p, UVM_PAGE_STATUS_CLEAN);
}
}
if (p->flags & PG_CLEAN) {
if (uvm_pagegetdirty(p) != UVM_PAGE_STATUS_DIRTY) {
int slot;
int pageidx;

111
sys/uvm/uvm_vnode.c
View File

@ -1,4 +1,4 @@
/* $NetBSD: uvm_vnode.c,v 1.104 2019/12/21 14:41:44 ad Exp $ */
/* $NetBSD: uvm_vnode.c,v 1.105 2020/01/15 17:55:45 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@ -45,7 +45,7 @@
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.104 2019/12/21 14:41:44 ad Exp $");
__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.105 2020/01/15 17:55:45 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
@ -66,6 +66,7 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.104 2019/12/21 14:41:44 ad Exp $");
#include <uvm/uvm.h>
#include <uvm/uvm_readahead.h>
#include <uvm/uvm_page_array.h>
#ifdef UVMHIST
UVMHIST_DEFINE(ubchist);
@ -82,7 +83,8 @@ static int uvn_put(struct uvm_object *, voff_t, voff_t, int);
static void uvn_reference(struct uvm_object *);
static int uvn_findpage(struct uvm_object *, voff_t, struct vm_page **,
int);
unsigned int, struct uvm_page_array *a,
unsigned int);
/*
* master pager structure
@ -136,7 +138,6 @@ uvn_detach(struct uvm_object *uobj)
*
* => object must be locked on entry! VOP_PUTPAGES must unlock it.
* => flags: PGO_SYNCIO -- use sync. I/O
* => note: caller must set PG_CLEAN and pmap_clear_modify (if needed)
*/
static int
@ -201,16 +202,23 @@ uvn_get(struct uvm_object *uobj, voff_t offset,
*/
int
uvn_findpages(struct uvm_object *uobj, voff_t offset, int *npagesp,
struct vm_page **pgs, int flags)
uvn_findpages(struct uvm_object *uobj, voff_t offset, unsigned int *npagesp,
struct vm_page **pgs, struct uvm_page_array *a, unsigned int flags)
{
int i, count, found, npages, rv;
unsigned int count, found, npages;
int i, rv;
struct uvm_page_array a_store;
if (a == NULL) {
a = &a_store;
uvm_page_array_init(a);
}
count = found = 0;
npages = *npagesp;
if (flags & UFP_BACKWARD) {
for (i = npages - 1; i >= 0; i--, offset -= PAGE_SIZE) {
rv = uvn_findpage(uobj, offset, &pgs[i], flags);
rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
i + 1);
if (rv == 0) {
if (flags & UFP_DIRTYONLY)
break;
@ -220,7 +228,8 @@ uvn_findpages(struct uvm_object *uobj, voff_t offset, int *npagesp,
}
} else {
for (i = 0; i < npages; i++, offset += PAGE_SIZE) {
rv = uvn_findpage(uobj, offset, &pgs[i], flags);
rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
npages - i);
if (rv == 0) {
if (flags & UFP_DIRTYONLY)
break;
@ -229,16 +238,29 @@ uvn_findpages(struct uvm_object *uobj, voff_t offset, int *npagesp,
count++;
}
}
if (a == &a_store) {
uvm_page_array_fini(a);
}
*npagesp = count;
return (found);
}
/*
* uvn_findpage: find a single page
*
* if a suitable page was found, put it in *pgp and return 1.
* otherwise return 0.
*/
static int
uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
int flags)
unsigned int flags, struct uvm_page_array *a, unsigned int nleft)
{
struct vm_page *pg;
bool dirty;
const unsigned int fillflags =
((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) |
((flags & UFP_DIRTYONLY) ?
(UVM_PAGE_ARRAY_FILL_DIRTY|UVM_PAGE_ARRAY_FILL_DENSE) : 0);
UVMHIST_FUNC("uvn_findpage"); UVMHIST_CALLED(ubchist);
UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx", (uintptr_t)uobj, offset,
0, 0);
@ -247,11 +269,35 @@ uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
if (*pgp != NULL) {
UVMHIST_LOG(ubchist, "dontcare", 0,0,0,0);
return 0;
goto skip_offset;
}
for (;;) {
/* look for an existing page */
pg = uvm_pagelookup(uobj, offset);
/*
* look for an existing page.
*
* XXX fragile API
* note that the array can be the one supplied by the caller of
* uvn_findpages. in that case, fillflags used by the caller
* might not match strictly with ours.
* in particular, the caller might have filled the array
* without DENSE but passed us UFP_DIRTYONLY (thus DENSE).
*/
pg = uvm_page_array_fill_and_peek(a, uobj, offset, nleft,
fillflags);
if (pg != NULL && pg->offset != offset) {
KASSERT(
((fillflags & UVM_PAGE_ARRAY_FILL_BACKWARD) != 0)
== (pg->offset < offset));
KASSERT(uvm_pagelookup(uobj, offset) == NULL
|| ((fillflags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0 &&
radix_tree_get_tag(&uobj->uo_pages,
offset >> PAGE_SHIFT, UVM_PAGE_DIRTY_TAG) == 0));
pg = NULL;
if ((fillflags & UVM_PAGE_ARRAY_FILL_DENSE) != 0) {
UVMHIST_LOG(ubchist, "dense", 0,0,0,0);
return 0;
}
}
/* nope? allocate one now */
if (pg == NULL) {
@ -268,28 +314,32 @@ uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
}
mutex_exit(uobj->vmobjlock);
uvm_wait("uvn_fp1");
uvm_page_array_clear(a);
mutex_enter(uobj->vmobjlock);
continue;
}
UVMHIST_LOG(ubchist, "alloced %#jx (color %ju)",
(uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
KASSERTMSG(uvm_pagegetdirty(pg) ==
UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg);
break;
} else if (flags & UFP_NOCACHE) {
UVMHIST_LOG(ubchist, "nocache",0,0,0,0);
return 0;
goto skip;
}
/* page is there, see if we need to wait on it */
if ((pg->flags & PG_BUSY) != 0) {
if (flags & UFP_NOWAIT) {
UVMHIST_LOG(ubchist, "nowait",0,0,0,0);
return 0;
goto skip;
}
pg->flags |= PG_WANTED;
UVMHIST_LOG(ubchist, "wait %#jx (color %ju)",
(uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
UVM_UNLOCK_AND_WAIT(pg, uobj->vmobjlock, 0,
"uvn_fp2", 0);
uvm_page_array_clear(a);
mutex_enter(uobj->vmobjlock);
continue;
}
@ -297,14 +347,12 @@ uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
/* skip PG_RDONLY pages if requested */
if ((flags & UFP_NORDONLY) && (pg->flags & PG_RDONLY)) {
UVMHIST_LOG(ubchist, "nordonly",0,0,0,0);
return 0;
goto skip;
}
/* stop on clean pages if requested */
if (flags & UFP_DIRTYONLY) {
dirty = pmap_clear_modify(pg) ||
(pg->flags & PG_CLEAN) == 0;
pg->flags |= PG_CLEAN;
const bool dirty = uvm_pagecheckdirty(pg, false);
if (!dirty) {
UVMHIST_LOG(ubchist, "dirtonly", 0,0,0,0);
return 0;
@ -316,10 +364,33 @@ uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
UVM_PAGE_OWN(pg, "uvn_findpage");
UVMHIST_LOG(ubchist, "found %#jx (color %ju)",
(uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
uvm_page_array_advance(a);
break;
}
*pgp = pg;
return 1;
skip_offset:
/*
* skip this offset
*/
pg = uvm_page_array_peek(a);
if (pg != NULL) {
if (pg->offset == offset) {
uvm_page_array_advance(a);
} else {
KASSERT((fillflags & UVM_PAGE_ARRAY_FILL_DENSE) == 0);
}
}
return 0;
skip:
/*
* skip this page
*/
KASSERT(pg != NULL);
uvm_page_array_advance(a);
return 0;
}
/*