NetBSD/sys/uvm/uvm_amap.c

/*	$NetBSD: uvm_amap.c,v 1.14 1998/08/31 02:43:14 thorpej Exp $	*/

/*
 * XXXCDC: "ROUGH DRAFT" QUALITY UVM PRE-RELEASE FILE!   
 *	   >>>USE AT YOUR OWN RISK, WORK IS NOT FINISHED<<<
 */
/*
 *
 * Copyright (c) 1997 Charles D. Cranor and Washington University.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by Charles D. Cranor and
 *      Washington University.
 * 4. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * from: Id: uvm_amap.c,v 1.1.2.25 1998/02/06 22:49:23 chs Exp
 */

/*
 * uvm_amap.c: uvm amap ops
 */

#include "opt_uvmhist.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/malloc.h>
#include <sys/pool.h>

#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_kern.h>

#define UVM_AMAP		/* pull in uvm_amap.h functions */
#include <uvm/uvm.h>
#include <uvm/uvm_swap.h>

/*
 * pool for vm_amap_structures.
 */

struct pool uvm_amap_pool;

/*
 * local functions
 */

static struct vm_amap *amap_alloc1 __P((int, int, int));

#ifdef VM_AMAP_PPREF
/*
 * what is ppref?   ppref is an _optional_ amap feature which is used
 * to keep track of reference counts on a per-page basis.  it is enabled
 * when VM_AMAP_PPREF is defined.
 *
 * when enabled, an array of ints is allocated for the pprefs.  this
 * array is allocated only when a partial reference is added to the
 * map (either by unmapping part of the amap, or gaining a reference
 * to only a part of an amap).  if the malloc of the array fails
 * (M_NOWAIT), then we set the array pointer to PPREF_NONE to indicate
 * that we tried to do ppref's but couldn't alloc the array so just
 * give up (after all, this is an optional feature!).
 *
 * the array is divided into page sized "chunks."   for chunks of length 1,
 * the chunk reference count plus one is stored in that chunk's slot.
 * for chunks of length > 1 the first slot contains (the reference count
 * plus one) * -1.    [the negative value indicates that the length is
 * greater than one.]   the second slot of the chunk contains the length
 * of the chunk.   here is an example:
 *
 * actual REFS:  2  2  2  2  3  1  1  0  0  0  4  4  0  1  1  1
 *       ppref: -3  4  x  x  4 -2  2 -1  3  x -5  2  1 -2  3  x
 *              <----------><-><----><-------><----><-><------->
 * (x = don't care)
 *
 * this allows us to allow one int to contain the ref count for the whole
 * chunk.    note that the "plus one" part is needed because a reference
 * count of zero is neither positive or negative (need a way to tell
 * if we've got one zero or a bunch of them).
 * 
 * here are some in-line functions to help us.
 */

static __inline void pp_getreflen __P((int *, int, int *, int *));
static __inline void pp_setreflen __P((int *, int, int, int));

/*
 * pp_getreflen: get the reference and length for a specific offset
 */
static __inline void
pp_getreflen(ppref, offset, refp, lenp)
	int *ppref, offset, *refp, *lenp;
{

	if (ppref[offset] > 0) {		/* chunk size must be 1 */
		*refp = ppref[offset] - 1;	/* don't forget to adjust */
		*lenp = 1;
	} else {
		*refp = (ppref[offset] * -1) - 1;
		*lenp = ppref[offset+1];
	}
}

/*
 * pp_setreflen: set the reference and length for a specific offset
 */
static __inline void
pp_setreflen(ppref, offset, ref, len)
	int *ppref, offset, ref, len;
{
	if (len == 1) {
		ppref[offset] = ref + 1;
	} else {
		ppref[offset] = (ref + 1) * -1;
		ppref[offset+1] = len;
	}
}
#endif

/*
 * amap_alloc1: internal function that allocates an amap, but does not
 *	init the overlay.
 *
 * => lock on returned amap is init'd
 */
static inline struct vm_amap *
amap_alloc1(slots, padslots, waitf)
	int slots, padslots, waitf;
{
	struct vm_amap *amap;
	int totalslots = slots + padslots;

	amap = pool_get(&uvm_amap_pool, (waitf == M_WAITOK) ? PR_WAITOK : 0);
	if (amap == NULL)
		return(NULL);

	simple_lock_init(&amap->am_l);
	amap->am_ref = 1;
	amap->am_flags = 0;
#ifdef VM_AMAP_PPREF
	amap->am_ppref = NULL;
#endif
	amap->am_maxslot = totalslots;
	amap->am_nslot = slots;
	amap->am_nused = 0;
	MALLOC(amap->am_slots,  int *, totalslots * sizeof(int), M_UVMAMAP, waitf);
	if (amap->am_slots) {
		MALLOC(amap->am_bckptr, int *, totalslots * sizeof(int), M_UVMAMAP, waitf);
		if (amap->am_bckptr) {
			MALLOC(amap->am_anon, struct vm_anon **, 
			    totalslots * sizeof(struct vm_anon *), M_UVMAMAP, waitf);
		}
	}

	if (amap->am_anon)
		return(amap);

	if (amap->am_slots) {
		FREE(amap->am_slots, M_UVMAMAP);
		if (amap->am_bckptr)
			FREE(amap->am_bckptr, M_UVMAMAP);
	}
	pool_put(&uvm_amap_pool, amap);
	return (NULL);
}

/*
 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM
 *
 * => caller should ensure sz is a multiple of PAGE_SIZE
 * => reference count to new amap is set to one
 * => new amap is returned unlocked
 */

struct vm_amap *
amap_alloc(sz, padsz, waitf)
	vaddr_t sz, padsz;
	int waitf;
{
	struct vm_amap *amap;
	int slots, padslots;
	UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist);

	AMAP_B2SLOT(slots, sz);		/* load slots */
	AMAP_B2SLOT(padslots, padsz);

	amap = amap_alloc1(slots, padslots, waitf);
	if (amap)
		memset(amap->am_anon, 0, (slots + padslots) * sizeof(struct vm_anon *));

	UVMHIST_LOG(maphist,"<- done, amap = 0x%x, sz=%d", amap, sz, 0, 0);
	return(amap);
}


/*
 * amap_free: free an amap
 *
 * => there should not be any valid references to the amap, so locking
 *	of the amap being freed is not an issue (doesn't matter).
 * => the amap is "gone" after we are done with it.
 */
void
amap_free(amap)
	struct vm_amap *amap;
{
	UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist);

#ifdef DIAGNOSTIC
	if (amap->am_ref || amap->am_nused)
		panic("amap_free");
#endif

	FREE(amap->am_slots, M_UVMAMAP);
	FREE(amap->am_bckptr, M_UVMAMAP);
	FREE(amap->am_anon, M_UVMAMAP);
#ifdef VM_AMAP_PPREF
	if (amap->am_ppref && amap->am_ppref != PPREF_NONE)
		FREE(amap->am_ppref, M_UVMAMAP);
#endif
	pool_put(&uvm_amap_pool, amap);

	UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0);
}

/*
 * amap_extend: extend the size of an amap (if needed)
 *
 * => amap being extended should be passed in unlocked (we will lock
 *	it as needed).
 * => amap has a reference count of one (our map entry)
 * => XXXCDC: should it have a waitflag???
 */
void
amap_extend(entry, addsize)
	vm_map_entry_t entry;
	vsize_t addsize;
{
	struct vm_amap *amap = entry->aref.ar_amap;
	int slotoff = entry->aref.ar_slotoff;
	int slotmapped, slotadd, slotneed;
#ifdef VM_AMAP_PPREF
	int *newppref, *oldppref;
#endif
	u_int *newsl, *newbck, *oldsl, *oldbck;
	struct vm_anon **newover, **oldover;
	int slotadded;
	UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist);

	UVMHIST_LOG(maphist, "  (entry=0x%x, addsize=0x%x)", entry,addsize,0,0);

	/*
	 * first, determine how many slots we need in the amap.   don't forget
	 * that ar_slotoff could be non-zero: this means that there are some
	 * unused slots before us in the amap.
	 */

	simple_lock(&amap->am_l);				/* lock! */

	AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */
	AMAP_B2SLOT(slotadd, addsize);			/* slots to add */
	slotneed = slotoff + slotmapped + slotadd;

	/*
	 * case 1: we already have enough slots in the map and thus only need
	 * to bump the reference counts on the slots we are adding.
	 */

	if (amap->am_nslot >= slotneed) {
#ifdef VM_AMAP_PPREF
		if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
			amap_pp_adjref(amap, slotoff + slotmapped, addsize, 1);
		}
#endif
		simple_unlock(&amap->am_l);
		UVMHIST_LOG(maphist,"<- done (case 1), amap = 0x%x, sltneed=%d", 
		    amap, slotneed, 0, 0);
		return;				/* done! */
	}

	/*
	 * case 2: we pre-allocated slots for use and we just need to bump
	 * nslot up to take account for these slots.
	 */
	if (amap->am_maxslot >= slotneed) {
#ifdef VM_AMAP_PPREF
		if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
			if ((slotoff + slotmapped) < amap->am_nslot)
				amap_pp_adjref(amap, slotoff + slotmapped, 
				    (amap->am_nslot - (slotoff + slotmapped)) *
				    PAGE_SIZE, 1);
			pp_setreflen(amap->am_ppref, amap->am_nslot, 1, 
			   slotneed - amap->am_nslot);
		}
#endif
		amap->am_nslot = slotneed;
		simple_unlock(&amap->am_l);
		/*
		 * no need to zero am_anon since that was done at alloc time and we
		 * never shrink an allocation.
		 */
		UVMHIST_LOG(maphist,"<- done (case 2), amap = 0x%x, slotneed=%d", 
		    amap, slotneed, 0, 0);
		return;
	}

	/*
	 * case 3: we need to malloc a new amap and copy all the amap data over
	 *
	 * XXX: should we pad out this allocation in hopes of avoid future case3
	 * extends?
	 * XXX: how about using kernel realloc?
	 *
	 * NOTE: we have the only map that has a reference to this amap locked.
	 * thus, no one else is going to try and change the amap while it is 
	 * unlocked (but we unlock just to be safe).
	 */

	simple_unlock(&amap->am_l);		/* unlock in case we sleep in malloc */
#ifdef VM_AMAP_PPREF
	newppref = NULL;
	if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
		MALLOC(newppref, int *, slotneed * sizeof(int), M_UVMAMAP,
		    M_NOWAIT);
		if (newppref == NULL) {
			/* give up if malloc fails */
			FREE(amap->am_ppref, M_UVMAMAP);
			    amap->am_ppref = PPREF_NONE;
		}
	}
#endif
	MALLOC(newsl, int *, slotneed * sizeof(int), M_UVMAMAP, M_WAITOK);
	MALLOC(newbck, int *, slotneed * sizeof(int), M_UVMAMAP, M_WAITOK);
	MALLOC(newover, struct vm_anon **, slotneed * sizeof(struct vm_anon *),
						   M_UVMAMAP, M_WAITOK);
	simple_lock(&amap->am_l);		/* re-lock! */

#ifdef DIAGNOSTIC
	if (amap->am_maxslot >= slotneed)
		panic("amap_extend: amap changed during malloc");
#endif

	/*
	 * now copy everything over to new malloc'd areas...
	 */

	slotadded = slotneed - amap->am_nslot;

	/* do am_slots */
	oldsl = amap->am_slots;
	memcpy(newsl, oldsl, sizeof(int) * amap->am_nused);
	amap->am_slots = newsl;

	/* do am_anon */
	oldover = amap->am_anon;
	memcpy(newover, oldover, sizeof(struct vm_anon *) * amap->am_nslot);
	memset(newover + amap->am_nslot, 0, sizeof(struct vm_anon *) * slotadded);
	amap->am_anon = newover;

	/* do am_bckptr */
	oldbck = amap->am_bckptr;
	memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot);
	memset(newbck + amap->am_nslot, 0, sizeof(int) * slotadded); /* XXX: needed? */
	amap->am_bckptr = newbck;

#ifdef VM_AMAP_PPREF
	/* do ppref */
	oldppref = amap->am_ppref;
	if (newppref) {
		memcpy(newppref, oldppref, sizeof(int) * amap->am_nslot);
		memset(newppref + amap->am_nslot, 0, sizeof(int) * slotadded);
		amap->am_ppref = newppref;
		if ((slotoff + slotmapped) < amap->am_nslot)
			amap_pp_adjref(amap, slotoff + slotmapped, 
			    (amap->am_nslot - (slotoff + slotmapped)) *
			    PAGE_SIZE, 1);
		pp_setreflen(newppref, amap->am_nslot, 1, slotadded);
	}
#endif

	/* update master values */
	amap->am_nslot = slotneed;
	amap->am_maxslot = slotneed;

	/* unlock */
	simple_unlock(&amap->am_l);

	/* and free */
	FREE(oldsl, M_UVMAMAP);
	FREE(oldbck, M_UVMAMAP);
	FREE(oldover, M_UVMAMAP);
#ifdef VM_AMAP_PPREF
	if (oldppref && oldppref != PPREF_NONE)
		FREE(oldppref, M_UVMAMAP);
#endif
	UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%x, slotneed=%d", 
	    amap, slotneed, 0, 0);
}

/*
 * amap_share_protect: change protection of an amap in a sharemap
 *
 * for sharemaps it is not possible to find all of the maps which
 * reference the sharemap (e.g. to remove or change a mapping).
 * in order to get around this (and support sharemaps) we use 
 * pmap_page_protect to change the protection on all mappings of the
 * page.   we traverse am_anon or am_slots depending on the current
 * state of the amap.
 *
 * => the map that entry belongs to must be locked by the caller.
 * => the amap pointed to by entry->aref.ar_amap must be locked by caller.
 * => the map should be locked before the amap (by the caller).
 */
void
amap_share_protect(entry, prot)
	vm_map_entry_t entry;
	vm_prot_t prot;
{
	struct vm_amap *amap = entry->aref.ar_amap;
	int slots, lcv, slot, stop;

	AMAP_B2SLOT(slots, (entry->end - entry->start));
	stop = entry->aref.ar_slotoff + slots;

	if (slots < amap->am_nused) {
		/* cheaper to traverse am_anon */
		for (lcv = entry->aref.ar_slotoff ; lcv < stop ; lcv++) {
			if (amap->am_anon[lcv] == NULL)
				continue;
			if (amap->am_anon[lcv]->u.an_page != NULL)
				pmap_page_protect(
				    PMAP_PGARG(amap->am_anon[lcv]->u.an_page),
				prot);
		}
		return;
	}

	/* cheaper to traverse am_slots */
	for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
		slot = amap->am_slots[lcv];
		if (slot < entry->aref.ar_slotoff || slot >= stop)
			continue;
		if (amap->am_anon[slot]->u.an_page != NULL)
			pmap_page_protect(
			    PMAP_PGARG(amap->am_anon[slot]->u.an_page), prot);
	}
	return;
}

/*
 * amap_wipeout: wipeout all anon's in an amap; then free the amap!
 *
 * => if amap is part of an active map entry, then the map that contains
 *	the map entry must be locked.
 * => amap's reference count should be one (the final reference).
 * => the amap must be locked by the caller.
 */

void
amap_wipeout(amap)
	struct vm_amap *amap;
{
	int lcv, slot;
	struct vm_anon *anon;
	UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist);
	UVMHIST_LOG(maphist,"(amap=0x%x)", amap, 0,0,0);

	for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {

		slot = amap->am_slots[lcv];
		anon = amap->am_anon[slot];

		if (anon == NULL || anon->an_ref == 0) 
			panic("amap_wipeout: corrupt amap");

		simple_lock(&anon->an_lock); /* lock anon */

		UVMHIST_LOG(maphist,"  processing anon 0x%x, ref=%d", anon, 
		    anon->an_ref, 0, 0);

		if (--anon->an_ref != 0) {
			simple_unlock(&anon->an_lock);
			continue;
		}

		/*
		 * we have last reference to a vm_anon.   free the vm_anon.
		 */
		uvm_anfree(anon);
	}

	/*
	 * now we free the map
	 */

	amap->am_nused = 0;
	amap->am_ref--;		/* drop final reference */
	amap_free(amap);
	UVMHIST_LOG(maphist,"<- done!", 0,0,0,0);
}

/*
 * amap_copy: ensure that a map entry's "needs_copy" flag is false
 *	by copying the amap if necessary.
 * 
 * => an entry with a null amap pointer will get a new (blank) one.
 * => the map that the map entry belongs to must be locked by caller.
 * => the amap currently attached to "entry" (if any) must be unlocked.
 * => if canchunk is true, then we may clip the entry into a chunk
 * => "startva" and "endva" are used only if canchunk is true.  they are
 *     used to limit chunking (e.g. if you have a large space that you
 *     know you are going to need to allocate amaps for, there is no point
 *     in allowing that to be chunked)
 */

void
amap_copy(map, entry, waitf, canchunk, startva, endva)
	vm_map_t map;
	vm_map_entry_t entry;
	int waitf;
	boolean_t canchunk;
	vaddr_t startva, endva;
{
	struct vm_amap *amap, *srcamap;
	int slots, lcv;
	vaddr_t chunksize;
	UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist);
	UVMHIST_LOG(maphist, "  (map=%p, entry=%p, waitf=%d)", map, entry, waitf, 0);

	/*
	 * is there a map to copy?   if not, create one from scratch.
	 */

	if (entry->aref.ar_amap == NULL) {

		/*
		 * check to see if we have a large amap that we can chunk.
		 * we align startva/endva to chunk-sized boundaries and then
		 * clip to them.
		 */

		if (canchunk && atop(entry->end - entry->start) >=
		    UVM_AMAP_LARGE) {
			/* convert slots to bytes */
			chunksize = UVM_AMAP_CHUNK * PAGE_SIZE;
			startva = (startva / chunksize) * chunksize;
			endva = roundup(endva, chunksize);
			UVMHIST_LOG(maphist, "  chunk amap ==> clip 0x%x->0x%x"
			    "to 0x%x->0x%x", entry->start, entry->end, startva,
			    endva);
			UVM_MAP_CLIP_START(map, entry, startva);
			/* watch out for endva wrap-around! */
			if (endva >= startva)
				UVM_MAP_CLIP_END(map, entry, endva);
		}

		UVMHIST_LOG(maphist, "<- done [creating new amap 0x%x->0x%x]", 
		entry->start, entry->end, 0, 0);
		entry->aref.ar_slotoff = 0;
		entry->aref.ar_amap = amap_alloc(entry->end - entry->start, 0,
		    waitf);
		if (entry->aref.ar_amap != NULL)
			entry->etype &= ~UVM_ET_NEEDSCOPY;
		return;
	}

	/*
	 * first check and see if we are the only map entry referencing the amap
	 * we currently have.   if so, then we can just take it over rather
	 * than copying it.
	 */

	if (entry->aref.ar_amap->am_ref == 1) {
		entry->etype &= ~UVM_ET_NEEDSCOPY;
		UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]",
		    0, 0, 0, 0);
		return;
	}

	/*
	 * looks like we need to copy the map.
	 */

	UVMHIST_LOG(maphist,"  amap=%p, ref=%d, must copy it", 
	    entry->aref.ar_amap, entry->aref.ar_amap->am_ref, 0, 0);
	AMAP_B2SLOT(slots, entry->end - entry->start);
	amap = amap_alloc1(slots, 0, waitf);
	if (amap == NULL) {
		UVMHIST_LOG(maphist, "  amap_alloc1 failed", 0,0,0,0);
		return;
	}
	srcamap = entry->aref.ar_amap;
	simple_lock(&srcamap->am_l);

	/*
	 * need to double check reference count now that we've got the src amap
	 * locked down.   (in which case we lost a reference while we were
	 * mallocing the new map).
	 */

	if (srcamap->am_ref == 1) {
		/*
		 * take over the old amap, get rid of the new one we just
		 * allocated.
		 */
		entry->etype &= ~UVM_ET_NEEDSCOPY;
		amap->am_ref--;		/* drop final reference to map */
		amap_free(amap);
		simple_unlock(&srcamap->am_l);
		return;
	}

	/*
	 * copy it now.
	 */

	UVMHIST_LOG(maphist, "  copying amap now",0, 0, 0, 0);
	for (lcv = 0 ; lcv < slots; lcv++) {
		amap->am_anon[lcv] =
		    srcamap->am_anon[entry->aref.ar_slotoff + lcv];
		if (amap->am_anon[lcv] == NULL)
			continue;
		simple_lock(&amap->am_anon[lcv]->an_lock);
		amap->am_anon[lcv]->an_ref++;
		simple_unlock(&amap->am_anon[lcv]->an_lock);
		amap->am_bckptr[lcv] = amap->am_nused;
		amap->am_slots[amap->am_nused] = lcv;
		amap->am_nused++;
	}

	/*
	 * drop our reference to the old amap (srcamap) and unlock.  we will
	 * not have the very last reference to srcamap so there is no need
	 * to worry about freeing it.
	 */

	srcamap->am_ref--;
	if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0)
		srcamap->am_flags &= ~AMAP_SHARED;   /* clear shared flag */
#ifdef VM_AMAP_PPREF
	if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) {
		amap_pp_adjref(srcamap, entry->aref.ar_slotoff, 
		    entry->end - entry->start, -1);
	}
#endif

	simple_unlock(&srcamap->am_l);

	/*
	 * install new amap.
	 */

	entry->aref.ar_slotoff = 0;
	entry->aref.ar_amap = amap;
	entry->etype &= ~UVM_ET_NEEDSCOPY;

	/*
	 * done!
	 */
	UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0);
}

/*
 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2)
 *
 *	called during fork(2) when the parent process has a wired map
 *	entry.   in that case we want to avoid write-protecting pages
 *	in the parent's map (e.g. like what you'd do for a COW page)
 *	so we resolve the COW here.
 *
 * => assume parent's entry was wired, thus all pages are resident.
 * => assume pages that are loaned out (loan_count) are already mapped
 *	read-only in all maps, and thus no need for us to worry about them
 * => assume both parent and child vm_map's are locked
 * => caller passes child's map/entry in to us
 * => if we run out of memory we will unlock the amap and sleep _with_ the
 *	parent and child vm_map's locked(!).    we have to do this since
 *	we are in the middle of a fork(2) and we can't let the parent
 *	map change until we are done copying all the map entrys.
 * => XXXCDC: out of memory should cause fork to fail, but there is
 *	currently no easy way to do this (needs fix)
 * => page queues must be unlocked (we may lock them)
 */

void
amap_cow_now(map, entry)
	struct vm_map *map;
	struct vm_map_entry *entry;
{
	struct vm_amap *amap = entry->aref.ar_amap;
	int lcv, slot;
	struct vm_anon *anon, *nanon;
	struct vm_page *pg, *npg;

	/*
	 * note that if we unlock the amap then we must ReStart the "lcv" for
	 * loop because some other process could reorder the anon's in the
	 * am_anon[] array on us while the lock is dropped.
	 */
ReStart:
	simple_lock(&amap->am_l);

	for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {

		/*
		 * get the page
		 */

		slot = amap->am_slots[lcv];
		anon = amap->am_anon[slot];
		simple_lock(&anon->an_lock);
		pg = anon->u.an_page;

		/*
		 * page must be resident since parent is wired
		 */

		if (pg == NULL)
		    panic("amap_cow_now: non-resident wired page in anon %p",
			anon);

		/*
		 * if the anon ref count is one and the page is not loaned,
		 * then we are safe (the child has exclusive access to the
		 * page).  if the page is loaned, then it must already be
		 * mapped read-only.
		 *
		 * we only need to get involved when these are not true.
		 * [note: if loan_count == 0, then the anon must own the page]
		 */

		if (anon->an_ref > 1 && pg->loan_count == 0) {

			/*
			 * if the page is busy then we have to unlock, wait for
			 * it and then restart.
			 */
			if (pg->flags & PG_BUSY) {
				pg->flags |= PG_WANTED;
				simple_unlock(&amap->am_l);
				UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, FALSE,
				    "cownow", 0);
				goto ReStart;
			}

			/*
			 * ok, time to do a copy-on-write to a new anon
			 */
			nanon = uvm_analloc();
			if (nanon)
				npg = uvm_pagealloc(NULL, 0, nanon);
			else
				npg = NULL;	/* XXX: quiet gcc warning */

			if (nanon == NULL || npg == NULL) {
				/* out of memory */
				/*
				 * XXXCDC: we should cause fork to fail, but
				 * we can't ...
				 */
				if (nanon)
					uvm_anfree(nanon);
				simple_unlock(&anon->an_lock);
				simple_unlock(&amap->am_l);
				uvm_wait("cownowpage");
				goto ReStart;
			}
	
			/*
			 * got it... now we can copy the data and replace anon
			 * with our new one...
			 */
			uvm_pagecopy(pg, npg);		/* old -> new */
			anon->an_ref--;			/* can't drop to zero */
			amap->am_anon[slot] = nanon;	/* replace */

			/*
			 * drop PG_BUSY on new page ... since we have had it's
			 * owner locked the whole time it can't be
			 * PG_RELEASED | PG_WANTED.
			 */
			npg->flags &= ~(PG_BUSY|PG_FAKE);
			UVM_PAGE_OWN(npg, NULL);
			uvm_lock_pageq();
			uvm_pageactivate(npg);
			uvm_unlock_pageq();
		}

		simple_unlock(&anon->an_lock);
		/*
		 * done with this anon, next ...!
		 */

	}	/* end of 'for' loop */

	return;
}

/*
 * amap_splitref: split a single reference into two seperate references
 *
 * => caller must lock map which is referencing the amap
 * => caller must not lock amap referenced (we will do it)
 */
void
amap_splitref(origref, splitref, offset)
	struct vm_aref *origref, *splitref;
	vaddr_t offset;
{
	int leftslots;
	UVMHIST_FUNC("amap_splitref"); UVMHIST_CALLED(maphist);

	AMAP_B2SLOT(leftslots, offset);
	if (leftslots == 0)
		panic("amap_splitref: split at zero offset");

	/*
	 * lock the amap
	 */
	simple_lock(&origref->ar_amap->am_l);

	/*
	 * now: amap is locked and we have a valid am_mapped array.
	 */

	if (origref->ar_amap->am_nslot - origref->ar_slotoff - leftslots <= 0)
		panic("amap_splitref: map size check failed");

	splitref->ar_amap = origref->ar_amap;
	splitref->ar_amap->am_ref++;		/* not a share reference */
	splitref->ar_slotoff = origref->ar_slotoff + leftslots;

	simple_unlock(&origref->ar_amap->am_l);
}

#ifdef VM_AMAP_PPREF

/*
 * amap_pp_establish: add a ppref array to an amap, if possible
 *
 * => amap locked by caller
 */
void
amap_pp_establish(amap)
	struct vm_amap *amap;
{

	MALLOC(amap->am_ppref, int *, sizeof(int) * amap->am_maxslot,
	    M_UVMAMAP, M_NOWAIT);

	/*
	 * if we fail then we just won't use ppref for this amap
	 */
	if (amap->am_ppref == NULL) {
		amap->am_ppref = PPREF_NONE;	/* not using it */
		return;
	}

	/*
	 * init ppref
	 */
	memset(amap->am_ppref, 0, sizeof(int) * amap->am_maxslot);
	pp_setreflen(amap->am_ppref, 0, amap->am_ref, amap->am_nslot);
	return;
}

/*
 * amap_pp_adjref: adjust reference count to a part of an amap using the
 * per-page reference count array.
 *
 * => map and amap locked by caller
 * => caller must check that ppref != PPREF_NONE before calling
 */
void
amap_pp_adjref(amap, curslot, bytelen, adjval)
	struct vm_amap *amap;
	int curslot;
	vsize_t bytelen;
	int adjval;
{
	int slots, stopslot, *ppref, lcv;
	int ref, len;

	/*
	 * get init values
	 */

	AMAP_B2SLOT(slots, bytelen);
	stopslot = curslot + slots;
	ppref = amap->am_ppref;

	/*
	 * first advance to the correct place in the ppref array, fragment
	 * if needed.
	 */

	for (lcv = 0 ; lcv < curslot ; lcv += len) {
		pp_getreflen(ppref, lcv, &ref, &len);
		if (lcv + len > curslot) {     /* goes past start? */
			pp_setreflen(ppref, lcv, ref, curslot - lcv);
			pp_setreflen(ppref, curslot, ref, len - (curslot -lcv));
			len = curslot - lcv;   /* new length of entry @ lcv */
		}
	}

	/*
	 * now adjust reference counts in range (make sure we dont overshoot)
	 */

	if (lcv != curslot)
		panic("ADJREF");

	for (/* lcv already set */; lcv < stopslot ; lcv += len) {
		pp_getreflen(ppref, lcv, &ref, &len);
		if (lcv + len > stopslot) {     /* goes past end? */
			pp_setreflen(ppref, lcv, ref, stopslot - lcv);
			pp_setreflen(ppref, stopslot, ref,
			    len - (stopslot - lcv));
			len = stopslot - lcv;
		}
		ref = ref + adjval;    /* ADJUST! */
		if (ref < 0)
			panic("amap_pp_adjref: negative reference count");
		pp_setreflen(ppref, lcv, ref, len);
		if (ref == 0)
			amap_wiperange(amap, lcv, len);
	}

}

/*
 * amap_wiperange: wipe out a range of an amap
 * [different from amap_wipeout because the amap is kept intact]
 *
 * => both map and amap must be locked by caller.
 */
void
amap_wiperange(amap, slotoff, slots)
	struct vm_amap *amap;
	int slotoff, slots;
{
	int byanon, lcv, stop, curslot, ptr;
	struct vm_anon *anon;
	UVMHIST_FUNC("amap_wiperange"); UVMHIST_CALLED(maphist);

	/*
	 * we can either traverse the amap by am_anon or by am_slots depending
	 * on which is cheaper.    decide now.
	 */

	if (slots < amap->am_nused) {
		byanon = TRUE;
		lcv = slotoff;
		stop = slotoff + slots;
	} else {
		byanon = FALSE;
		lcv = 0;
		stop = amap->am_nused;
	}

	/*
	 * ok, now do it!
	 */

	for (; lcv < stop; lcv++) {

		/*
		 * verify the anon is ok.
		 */
		if (byanon) {
			if (amap->am_anon[lcv] == NULL)
				continue;
			curslot = lcv;
		} else {
			curslot = amap->am_slots[lcv];
			if (curslot < slotoff || curslot >= stop)
				continue;
		}
		anon = amap->am_anon[curslot];

		/*
		 * remove it from the amap
		 */
		amap->am_anon[curslot] = NULL;
		ptr = amap->am_bckptr[curslot];
		if (ptr != (amap->am_nused - 1)) {
			amap->am_slots[ptr] =
			    amap->am_slots[amap->am_nused - 1];
			amap->am_bckptr[amap->am_slots[ptr]] =
			    ptr;    /* back ptr. */
		}
		amap->am_nused--;

		/*
		 * drop anon reference count
		 */
		simple_lock(&anon->an_lock);
		if (--anon->an_ref != 0) {
			simple_unlock(&anon->an_lock);
			continue;
		}

		/*
		 * we just eliminated the last reference to an anon.   free it.
		 */
		uvm_anfree(anon);
	}
}

#endif

/*
 * allocate anons
 */
void
uvm_anon_init()
{
	struct vm_anon *anon;
	int nanon = uvmexp.free - (uvmexp.free / 16); /* XXXCDC ??? */
	int lcv;

	/*
	 * Initialize the vm_amap pool.
	 */
	pool_init(&uvm_amap_pool, sizeof(struct vm_amap), 0, 0, 0,
	    "amappl", 0,
	    pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAMAP);

	/*
	 * Allocate the initial anons.
	 */
	anon = (struct vm_anon *)uvm_km_alloc(kernel_map,
	    sizeof(*anon) * nanon);
	if (anon == NULL) {
		printf("uvm_anon_init: can not allocate %d anons\n", nanon);
		panic("uvm_anon_init");
	}

	memset(anon, 0, sizeof(*anon) * nanon);
	uvm.afree = NULL;
	uvmexp.nanon = uvmexp.nfreeanon = nanon;
	for (lcv = 0 ; lcv < nanon ; lcv++) {
		anon[lcv].u.an_nxt = uvm.afree;
		uvm.afree = &anon[lcv];
	}
	simple_lock_init(&uvm.afreelock);
}

/*
 * add some more anons to the free pool.  called when we add
 * more swap space.
 */
void
uvm_anon_add(pages)
	int	pages;
{
	struct vm_anon *anon;
	int lcv;

	anon = (struct vm_anon *)uvm_km_alloc(kernel_map,
	    sizeof(*anon) * pages);

	/* XXX Should wait for VM to free up. */
	if (anon == NULL) {
		printf("uvm_anon_add: can not allocate %d anons\n", pages);
		panic("uvm_anon_add");
	}

	simple_lock(&uvm.afreelock);
	memset(anon, 0, sizeof(*anon) * pages);
	uvmexp.nanon += pages;
	uvmexp.nfreeanon += pages;
	for (lcv = 0; lcv < pages; lcv++) {
		anon[lcv].u.an_nxt = uvm.afree;
		uvm.afree = &anon[lcv];
	}
	simple_unlock(&uvm.afreelock);
}

/*
 * allocate an anon
 */
struct vm_anon *
uvm_analloc()
{
	struct vm_anon *a;

	simple_lock(&uvm.afreelock);
	a = uvm.afree;
	if (a) {
		uvm.afree = a->u.an_nxt;
		uvmexp.nfreeanon--;
		a->an_ref = 1;
		a->an_swslot = 0;
		a->u.an_page = NULL;		/* so we can free quickly */
		simple_lock_init(&a->an_lock);
	}
	simple_unlock(&uvm.afreelock);
	return(a);
}

/*
 * uvm_anfree: free a single anon structure
 *
 * => caller must remove anon from its amap before calling (if it was in
 *	an amap).
 * => if anon was in use, then it must be locked by the caller and the
 *	caller must have dropped the reference count to zero.
 * => we may lock the pageq's.
 */
void
uvm_anfree(anon)
	struct vm_anon *anon;
{
	struct vm_page *pg;
	UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist);
	UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0);

	/*
	 * get page
	 */

	pg = anon->u.an_page;

	/*
	 * if there is a resident page and it is loaned, then anon may not
	 * own it.   call out to uvm_anon_lockpage() to ensure the real owner
 	 * of the page has been identified and locked.
	 */

	if (pg && pg->loan_count)
		pg = uvm_anon_lockloanpg(anon);

	/*
	 * if we have a resident page, we must dispose of it before freeing
	 * the anon.
	 */

	if (pg) {

		/*
		 * if the page is owned by a uobject (now locked), then we must 
		 * kill the loan on the page rather than free it.
		 */

		if (pg->uobject) {

			/* kill loan */
			uvm_lock_pageq();
#ifdef DIAGNOSTIC
			if (pg->loan_count < 1)
		panic("uvm_anfree: obj owned page with no loan count");
#endif
			pg->loan_count--;
			pg->uanon = NULL;
			uvm_unlock_pageq();
			simple_unlock(&pg->uobject->vmobjlock);

		} else {

			/*
			 * page has no uobject, so we must be the owner of it.
			 *
			 * if page is busy then we just mark it as released
			 * (who ever has it busy must check for this when they
			 * wake up).    if the page is not busy then we can
			 * free it now.
			 */

			if ((pg->flags & PG_BUSY) != 0) {
				/* tell them to dump it when done */
				pg->flags |= PG_RELEASED;
				simple_unlock(&anon->an_lock);
				UVMHIST_LOG(maphist,
				    "  anon 0x%x, page 0x%x: BUSY (released!)", 
				    anon, pg, 0, 0);
				return;
			} 

			pmap_page_protect(PMAP_PGARG(pg), VM_PROT_NONE);
			uvm_lock_pageq();	/* lock out pagedaemon */
			uvm_pagefree(pg);	/* bye bye */
			uvm_unlock_pageq();	/* free the daemon */

			UVMHIST_LOG(maphist,"  anon 0x%x, page 0x%x: freed now!", 
			    anon, pg, 0, 0);
		}
	}

	/*
	 * are we using any backing store resources?   if so, free them.
	 */
	if (anon->an_swslot) {
		/*
		 * on backing store: no I/O in progress.  sole amap reference
		 * is ours and we've got it locked down.   thus we can free,
		 * and be done.
		 */
		UVMHIST_LOG(maphist,"  freeing anon 0x%x, paged to swslot 0x%x",
		    anon, anon->an_swslot, 0, 0);
		uvm_swap_free(anon->an_swslot, 1);
		anon->an_swslot = 0;
	} 

	/*
	 * now that we've stripped the data areas from the anon, free the anon
	 * itself!
	 */
	simple_lock(&uvm.afreelock);
	anon->u.an_nxt = uvm.afree;
	uvm.afree = anon;
	uvmexp.nfreeanon++;
	simple_unlock(&uvm.afreelock);
	UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
}

/*
 * uvm_anon_lockloanpg: given a locked anon, lock its resident page
 *
 * => anon is locked by caller
 * => on return: anon is locked
 *		 if there is a resident page:
 *			if it has a uobject, it is locked by us
 *			if it is ownerless, we take over as owner
 *		 we return the resident page (it can change during
 *		 this function)
 * => note that the only time an anon has an ownerless resident page
 *	is if the page was loaned from a uvm_object and the uvm_object
 *	disowned it
 * => this only needs to be called when you want to do an operation
 *	on an anon's resident page and that page has a non-zero loan
 *	count.
 */
struct vm_page *
uvm_anon_lockloanpg(anon)
	struct vm_anon *anon;
{
	struct vm_page *pg;
	boolean_t locked = FALSE;

	/*
	 * loop while we have a resident page that has a non-zero loan count.
	 * if we successfully get our lock, we will "break" the loop.
	 * note that the test for pg->loan_count is not protected -- this
	 * may produce false positive results.   note that a false positive
	 * result may cause us to do more work than we need to, but it will
	 * not produce an incorrect result.
	 */

	while (((pg = anon->u.an_page) != NULL) && pg->loan_count != 0) {

		/*
		 * quickly check to see if the page has an object before
		 * bothering to lock the page queues.   this may also produce
		 * a false positive result, but that's ok because we do a real
		 * check after that.
		 *
		 * XXX: quick check -- worth it?   need volatile?
		 */

		if (pg->uobject) {

			uvm_lock_pageq();
			if (pg->uobject) {	/* the "real" check */
				locked =
				    simple_lock_try(&pg->uobject->vmobjlock);
			} else {
				/* object disowned before we got PQ lock */
				locked = TRUE;
			}
			uvm_unlock_pageq();

			/*
			 * if we didn't get a lock (try lock failed), then we
			 * toggle our anon lock and try again
			 */

			if (!locked) {
				simple_unlock(&anon->an_lock);
				/*
				 * someone locking the object has a chance to
				 * lock us right now
				 */
				simple_lock(&anon->an_lock);
				continue;		/* start over */
			}
		}

		/*
		 * if page is un-owned [i.e. the object dropped its ownership],
		 * then we can take over as owner!
		 */

		if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) {
			uvm_lock_pageq();
			pg->pqflags |= PQ_ANON;		/* take ownership... */
			pg->loan_count--;	/* ... and drop our loan */
			uvm_unlock_pageq();
		}

		/*
		 * we did it!   break the loop
		 */
		break;
	}

	/*
	 * done!
	 */

	return(pg);
}