NetBSD/sys/uvm/uvm_vnode.c

/*	$NetBSD: uvm_vnode.c,v 1.121 2024/04/05 13:05:41 riastradh Exp $	*/

/*
 * Copyright (c) 1997 Charles D. Cranor and Washington University.
 * Copyright (c) 1991, 1993
 *      The Regents of the University of California.
 * Copyright (c) 1990 University of Utah.
 *
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * the Systems Programming Group of the University of Utah Computer
 * Science Department.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)vnode_pager.c       8.8 (Berkeley) 2/13/94
 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp
 */

/*
 * uvm_vnode.c: the vnode pager.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.121 2024/04/05 13:05:41 riastradh Exp $");

#ifdef _KERNEL_OPT
#include "opt_uvmhist.h"
#endif

#include <sys/atomic.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/vnode.h>
#include <sys/disklabel.h>
#include <sys/ioctl.h>
#include <sys/fcntl.h>
#include <sys/conf.h>
#include <sys/pool.h>
#include <sys/mount.h>

#include <miscfs/specfs/specdev.h>

#include <uvm/uvm.h>
#include <uvm/uvm_readahead.h>
#include <uvm/uvm_page_array.h>

#ifdef UVMHIST
UVMHIST_DEFINE(ubchist);
#endif

/*
 * functions
 */

static void	uvn_alloc_ractx(struct uvm_object *);
static void	uvn_detach(struct uvm_object *);
static int	uvn_get(struct uvm_object *, voff_t, struct vm_page **, int *,
			int, vm_prot_t, int, int);
static void	uvn_markdirty(struct uvm_object *);
static int	uvn_put(struct uvm_object *, voff_t, voff_t, int);
static void	uvn_reference(struct uvm_object *);

static int	uvn_findpage(struct uvm_object *, voff_t, struct vm_page **,
			     unsigned int, struct uvm_page_array *a,
			     unsigned int);

/*
 * master pager structure
 */

const struct uvm_pagerops uvm_vnodeops = {
	.pgo_reference = uvn_reference,
	.pgo_detach = uvn_detach,
	.pgo_get = uvn_get,
	.pgo_put = uvn_put,
	.pgo_markdirty = uvn_markdirty,
};

/*
 * the ops!
 */

/*
 * uvn_reference
 *
 * duplicate a reference to a VM object.  Note that the reference
 * count must already be at least one (the passed in reference) so
 * there is no chance of the uvn being killed or locked out here.
 *
 * => caller must call with object unlocked.
 * => caller must be using the same accessprot as was used at attach time
 */

static void
uvn_reference(struct uvm_object *uobj)
{
	vref((struct vnode *)uobj);
}


/*
 * uvn_detach
 *
 * remove a reference to a VM object.
 *
 * => caller must call with object unlocked and map locked.
 */

static void
uvn_detach(struct uvm_object *uobj)
{
	vrele((struct vnode *)uobj);
}

/*
 * uvn_put: flush page data to backing store.
 *
 * => object must be locked on entry!   VOP_PUTPAGES must unlock it.
 * => flags: PGO_SYNCIO -- use sync. I/O
 */

static int
uvn_put(struct uvm_object *uobj, voff_t offlo, voff_t offhi, int flags)
{
	struct vnode *vp = (struct vnode *)uobj;
	int error;

	KASSERT(rw_write_held(uobj->vmobjlock));
	error = VOP_PUTPAGES(vp, offlo, offhi, flags);

	return error;
}

/*
 * uvn_get: get pages (synchronously) from backing store
 *
 * => prefer map unlocked (not required)
 * => object must be locked!  we will _unlock_ it before starting any I/O.
 * => flags: PGO_LOCKED: fault data structures are locked
 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
 * => NOTE: caller must check for released pages!!
 */

static int
uvn_get(struct uvm_object *uobj, voff_t offset,
    struct vm_page **pps /* IN/OUT */,
    int *npagesp /* IN (OUT if PGO_LOCKED)*/,
    int centeridx, vm_prot_t access_type, int advice, int flags)
{
	struct vnode *vp = (struct vnode *)uobj;
	int error;

	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)vp, offset,
	    0, 0);

	if (vp->v_type == VREG && (access_type & VM_PROT_WRITE) == 0
	    && (flags & PGO_LOCKED) == 0 && vp->v_tag != VT_TMPFS) {
		uvn_alloc_ractx(uobj);
		uvm_ra_request(vp->v_ractx, advice, uobj, offset,
		    *npagesp << PAGE_SHIFT);
	}

	error = VOP_GETPAGES(vp, offset, pps, npagesp, centeridx,
			     access_type, advice, flags);

	if (flags & PGO_LOCKED)
		KASSERT(rw_lock_held(uobj->vmobjlock));
	return error;
}

/*
 * uvn_markdirty: called when the object gains first dirty page
 *
 * => uobj must be write locked.
 */

static void
uvn_markdirty(struct uvm_object *uobj)
{
	struct vnode *vp = (struct vnode *)uobj;

	KASSERT(rw_write_held(uobj->vmobjlock));

	mutex_enter(vp->v_interlock);
	if ((vp->v_iflag & VI_ONWORKLST) == 0) {
		vn_syncer_add_to_worklist(vp, filedelay);
	}
	mutex_exit(vp->v_interlock);
}

/*
 * uvn_findpages:
 * return the page for the uobj and offset requested, allocating if needed.
 * => uobj must be locked.
 * => returned pages will be BUSY.
 */

int
uvn_findpages(struct uvm_object *uobj, voff_t offset, unsigned int *npagesp,
    struct vm_page **pgs, struct uvm_page_array *a, unsigned int flags)
{
	unsigned int count, found, npages;
	int i, rv;
	struct uvm_page_array a_store;

	if (a == NULL) {
		/*
		 * XXX fragile API
		 * note that the array can be the one supplied by the caller of
		 * uvn_findpages.  in that case, fillflags used by the caller
		 * might not match strictly with ours.
		 * in particular, the caller might have filled the array
		 * without DENSE but passed us UFP_DIRTYONLY (thus DENSE).
		 */
		const unsigned int fillflags =
		    ((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) |
		    ((flags & UFP_DIRTYONLY) ?
		    (UVM_PAGE_ARRAY_FILL_DIRTY|UVM_PAGE_ARRAY_FILL_DENSE) : 0);
		a = &a_store;
		uvm_page_array_init(a, uobj, fillflags);
	}
	count = found = 0;
	npages = *npagesp;
	if (flags & UFP_BACKWARD) {
		for (i = npages - 1; i >= 0; i--, offset -= PAGE_SIZE) {
			rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
			    i + 1);
			if (rv == 0) {
				if (flags & UFP_DIRTYONLY)
					break;
			} else
				found++;
			count++;
		}
	} else {
		for (i = 0; i < npages; i++, offset += PAGE_SIZE) {
			rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
			    npages - i);
			if (rv == 0) {
				if (flags & UFP_DIRTYONLY)
					break;
			} else
				found++;
			count++;
		}
	}
	if (a == &a_store) {
		uvm_page_array_fini(a);
	}
	*npagesp = count;
	return (found);
}

/*
 * uvn_findpage: find a single page
 *
 * if a suitable page was found, put it in *pgp and return 1.
 * otherwise return 0.
 */

static int
uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
    unsigned int flags, struct uvm_page_array *a, unsigned int nleft)
{
	struct vm_page *pg;
	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)uobj, offset,
	    0, 0);

	/*
	 * NOBUSY must come with NOWAIT and NOALLOC.  if NOBUSY is
	 * specified, this may be called with a reader lock.
	 */

	KASSERT(rw_lock_held(uobj->vmobjlock));
	KASSERT((flags & UFP_NOBUSY) == 0 || (flags & UFP_NOWAIT) != 0);
	KASSERT((flags & UFP_NOBUSY) == 0 || (flags & UFP_NOALLOC) != 0);
	KASSERT((flags & UFP_NOBUSY) != 0 || rw_write_held(uobj->vmobjlock));

	if (*pgp != NULL) {
		UVMHIST_LOG(ubchist, "dontcare", 0,0,0,0);
		goto skip_offset;
	}
	for (;;) {
		/*
		 * look for an existing page.
		 */
		pg = uvm_page_array_fill_and_peek(a, offset, nleft);
		if (pg != NULL && pg->offset != offset) {
			struct vm_page __diagused *tpg;
			KASSERT(
			    ((a->ar_flags & UVM_PAGE_ARRAY_FILL_BACKWARD) != 0)
			    == (pg->offset < offset));
			KASSERT((tpg = uvm_pagelookup(uobj, offset)) == NULL ||
				((a->ar_flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0 &&
				 !uvm_obj_page_dirty_p(tpg)));
			pg = NULL;
			if ((a->ar_flags & UVM_PAGE_ARRAY_FILL_DENSE) != 0) {
				UVMHIST_LOG(ubchist, "dense", 0,0,0,0);
				return 0;
			}
		}

		/* nope?  allocate one now */
		if (pg == NULL) {
			if (flags & UFP_NOALLOC) {
				UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0);
				return 0;
			}
			pg = uvm_pagealloc(uobj, offset, NULL,
			    UVM_FLAG_COLORMATCH);
			if (pg == NULL) {
				if (flags & UFP_NOWAIT) {
					UVMHIST_LOG(ubchist, "nowait",0,0,0,0);
					return 0;
				}
				rw_exit(uobj->vmobjlock);
				uvm_wait("uvnfp1");
				uvm_page_array_clear(a);
				rw_enter(uobj->vmobjlock, RW_WRITER);
				continue;
			}
			UVMHIST_LOG(ubchist, "alloced %#jx (color %ju)",
			    (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
			KASSERTMSG(uvm_pagegetdirty(pg) ==
			    UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg);
			break;
		} else if (flags & UFP_NOCACHE) {
			UVMHIST_LOG(ubchist, "nocache",0,0,0,0);
			goto skip;
		}

		/* page is there, see if we need to wait on it */
		if ((pg->flags & PG_BUSY) != 0) {
			if (flags & UFP_NOWAIT) {
				UVMHIST_LOG(ubchist, "nowait",0,0,0,0);
				goto skip;
			}
			UVMHIST_LOG(ubchist, "wait %#jx (color %ju)",
			    (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
			uvm_pagewait(pg, uobj->vmobjlock, "uvnfp2");
			uvm_page_array_clear(a);
			rw_enter(uobj->vmobjlock, RW_WRITER);
			continue;
		}

		/* skip PG_RDONLY pages if requested */
		if ((flags & UFP_NORDONLY) && (pg->flags & PG_RDONLY)) {
			UVMHIST_LOG(ubchist, "nordonly",0,0,0,0);
			goto skip;
		}

		/* stop on clean pages if requested */
		if (flags & UFP_DIRTYONLY) {
			const bool dirty = uvm_pagecheckdirty(pg, false);
			if (!dirty) {
				UVMHIST_LOG(ubchist, "dirtonly", 0,0,0,0);
				return 0;
			}
		}

		/* mark the page BUSY and we're done. */
		if ((flags & UFP_NOBUSY) == 0) {
			pg->flags |= PG_BUSY;
			UVM_PAGE_OWN(pg, "uvn_findpage");
		}
		UVMHIST_LOG(ubchist, "found %#jx (color %ju)",
		    (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0);
		uvm_page_array_advance(a);
		break;
	}
	*pgp = pg;
	return 1;

 skip_offset:
	/*
	 * skip this offset
	 */
	pg = uvm_page_array_peek(a);
	if (pg != NULL) {
		if (pg->offset == offset) {
			uvm_page_array_advance(a);
		} else {
			KASSERT((a->ar_flags & UVM_PAGE_ARRAY_FILL_DENSE) == 0);
		}
	}
	return 0;

 skip:
	/*
	 * skip this page
	 */
	KASSERT(pg != NULL);
	uvm_page_array_advance(a);
	return 0;
}

/*
 * uvm_vnp_setsize: grow or shrink a vnode uobj
 *
 * grow   => just update size value
 * shrink => toss un-needed pages
 *
 * => we assume that the caller has a reference of some sort to the
 *	vnode in question so that it will not be yanked out from under
 *	us.
 */

void
uvm_vnp_setsize(struct vnode *vp, voff_t newsize)
{
	struct uvm_object *uobj = &vp->v_uobj;
	voff_t pgend = round_page(newsize);
	voff_t oldsize;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);

	rw_enter(uobj->vmobjlock, RW_WRITER);
	UVMHIST_LOG(ubchist, "vp %#jx old %#jx new %#jx",
	    (uintptr_t)vp, vp->v_size, newsize, 0);

	/*
	 * now check if the size has changed: if we shrink we had better
	 * toss some pages...
	 */

	KASSERT(newsize != VSIZENOTSET);
	KASSERT(newsize >= 0);
	KASSERTMSG(vp->v_size <= vp->v_writesize, "vp=%p"
	    " v_size=0x%llx v_writesize=0x%llx", vp,
	    (unsigned long long)vp->v_size,
	    (unsigned long long)vp->v_writesize);
	KASSERTMSG((vp->v_size == vp->v_writesize ||
		newsize == vp->v_writesize || newsize <= vp->v_size),
	    "vp=%p v_size=0x%llx v_writesize=0x%llx newsize=0x%llx",
	    vp,
	    (unsigned long long)vp->v_size,
	    (unsigned long long)vp->v_writesize,
	    (unsigned long long)newsize);

	oldsize = vp->v_writesize;

	/*
	 * check whether size shrinks
	 * if old size hasn't been set, there are no pages to drop
	 * if there was an integer overflow in pgend, then this is no shrink
	 */
	if (oldsize > pgend && oldsize != VSIZENOTSET && pgend >= 0) {
		(void) uvn_put(uobj, pgend, 0, PGO_FREE | PGO_SYNCIO);
		rw_enter(uobj->vmobjlock, RW_WRITER);
	}
	mutex_enter(vp->v_interlock);
	vp->v_size = vp->v_writesize = newsize;
	mutex_exit(vp->v_interlock);
	rw_exit(uobj->vmobjlock);
}

void
uvm_vnp_setwritesize(struct vnode *vp, voff_t newsize)
{

	rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
	KASSERT(newsize != VSIZENOTSET);
	KASSERT(newsize >= 0);
	KASSERT(vp->v_size != VSIZENOTSET);
	KASSERT(vp->v_writesize != VSIZENOTSET);
	KASSERTMSG(vp->v_size <= vp->v_writesize, "vp=%p"
	    " v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", vp,
	    (unsigned long long)vp->v_size,
	    (unsigned long long)vp->v_writesize,
	    (unsigned long long)newsize);
	KASSERTMSG(vp->v_size <= newsize, "vp=%p"
	    " v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", vp,
	    (unsigned long long)vp->v_size,
	    (unsigned long long)vp->v_writesize,
	    (unsigned long long)newsize);
	mutex_enter(vp->v_interlock);
	vp->v_writesize = newsize;
	mutex_exit(vp->v_interlock);
	rw_exit(vp->v_uobj.vmobjlock);
}

bool
uvn_text_p(struct uvm_object *uobj)
{
	struct vnode *vp = (struct vnode *)uobj;
	int iflag;

	/*
	 * v_interlock is not held here, but VI_EXECMAP is only ever changed
	 * with the vmobjlock held too.
	 */
	iflag = atomic_load_relaxed(&vp->v_iflag);
	return (iflag & VI_EXECMAP) != 0;
}

static void
uvn_alloc_ractx(struct uvm_object *uobj)
{
	struct vnode *vp = (struct vnode *)uobj;
	struct uvm_ractx *ra = NULL;

	KASSERT(rw_write_held(uobj->vmobjlock));

	if (vp->v_type != VREG) {
		return;
	}
	if (vp->v_ractx != NULL) {
		return;
	}
	if (vp->v_ractx == NULL) {
		rw_exit(uobj->vmobjlock);
		ra = uvm_ra_allocctx();
		rw_enter(uobj->vmobjlock, RW_WRITER);
		if (ra != NULL && vp->v_ractx == NULL) {
			vp->v_ractx = ra;
			ra = NULL;
		}
	}
	if (ra != NULL) {
		uvm_ra_freectx(ra);
	}
}