NetBSD/sys/uvm/uvm_glue.c

696 lines
18 KiB
C
Raw Normal View History

/* $NetBSD: uvm_glue.c,v 1.59 2002/07/02 20:27:48 yamt Exp $ */
2001-05-25 08:06:11 +04:00
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
2001-05-25 08:06:11 +04:00
* Copyright (c) 1991, 1993, The Regents of the University of California.
*
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* The Mach Operating System project at Carnegie-Mellon University.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by Charles D. Cranor,
2001-05-25 08:06:11 +04:00
* Washington University, the University of California, Berkeley and
* its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vm_glue.c 8.6 (Berkeley) 1/5/94
1998-02-07 14:07:38 +03:00
* from: Id: uvm_glue.c,v 1.1.2.8 1998/02/07 01:16:54 chs Exp
*
*
* Copyright (c) 1987, 1990 Carnegie-Mellon University.
* All rights reserved.
2001-05-25 08:06:11 +04:00
*
* Permission to use, copy, modify and distribute this software and
* its documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
2001-05-25 08:06:11 +04:00
*
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
* FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
2001-05-25 08:06:11 +04:00
*
* Carnegie Mellon requests users of this software to return to
*
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
*
* any improvements or extensions that they make and grant Carnegie the
* rights to redistribute these changes.
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_glue.c,v 1.59 2002/07/02 20:27:48 yamt Exp $");
2001-05-30 19:24:23 +04:00
#include "opt_kgdb.h"
#include "opt_kstack.h"
1998-10-20 02:21:19 +04:00
#include "opt_sysv.h"
2001-05-30 19:24:23 +04:00
#include "opt_uvmhist.h"
/*
* uvm_glue.c: glue functions
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/buf.h>
#include <sys/user.h>
#ifdef SYSVSHM
#include <sys/shm.h>
#endif
#include <uvm/uvm.h>
#include <machine/cpu.h>
/*
* local prototypes
*/
static void uvm_swapout __P((struct proc *));
/*
* XXXCDC: do these really belong here?
*/
int readbuffers = 0; /* allow KGDB to read kern buffer pool */
/* XXX: see uvm_kernacc */
/*
* uvm_kernacc: can the kernel access a region of memory
*
* - called from malloc [DIAGNOSTIC], and /dev/kmem driver (mem.c)
*/
1998-03-09 03:58:55 +03:00
boolean_t
uvm_kernacc(addr, len, rw)
caddr_t addr;
size_t len;
int rw;
{
1998-03-09 03:58:55 +03:00
boolean_t rv;
vaddr_t saddr, eaddr;
1998-03-09 03:58:55 +03:00
vm_prot_t prot = rw == B_READ ? VM_PROT_READ : VM_PROT_WRITE;
saddr = trunc_page((vaddr_t)addr);
eaddr = round_page((vaddr_t)addr + len);
1998-03-09 03:58:55 +03:00
vm_map_lock_read(kernel_map);
rv = uvm_map_checkprot(kernel_map, saddr, eaddr, prot);
vm_map_unlock_read(kernel_map);
/*
* XXX there are still some things (e.g. the buffer cache) that
* are managed behind the VM system's back so even though an
* address is accessible in the mind of the VM system, there may
* not be physical pages where the VM thinks there is. This can
* lead to bogus allocation of pages in the kernel address space
* or worse, inconsistencies at the pmap level. We only worry
* about the buffer cache for now.
*/
if (!readbuffers && rv && (eaddr > (vaddr_t)buffers &&
saddr < (vaddr_t)buffers + MAXBSIZE * nbuf))
1998-03-09 03:58:55 +03:00
rv = FALSE;
return(rv);
}
/*
* uvm_useracc: can the user access it?
*
* - called from physio() and sys___sysctl().
*/
1998-03-09 03:58:55 +03:00
boolean_t
uvm_useracc(addr, len, rw)
caddr_t addr;
size_t len;
int rw;
{
struct vm_map *map;
1998-03-09 03:58:55 +03:00
boolean_t rv;
vm_prot_t prot = rw == B_READ ? VM_PROT_READ : VM_PROT_WRITE;
/* XXX curproc */
map = &curproc->p_vmspace->vm_map;
vm_map_lock_read(map);
rv = uvm_map_checkprot(map, trunc_page((vaddr_t)addr),
round_page((vaddr_t)addr + len), prot);
vm_map_unlock_read(map);
1998-03-09 03:58:55 +03:00
return(rv);
}
#ifdef KGDB
/*
* Change protections on kernel pages from addr to addr+len
* (presumably so debugger can plant a breakpoint).
*
* We force the protection change at the pmap level. If we were
* to use vm_map_protect a change to allow writing would be lazily-
* applied meaning we would still take a protection fault, something
* we really don't want to do. It would also fragment the kernel
* map unnecessarily. We cannot use pmap_protect since it also won't
* enforce a write-enable request. Using pmap_enter is the only way
* we can ensure the change takes place properly.
*/
1998-03-09 03:58:55 +03:00
void
uvm_chgkprot(addr, len, rw)
2000-03-30 16:31:50 +04:00
caddr_t addr;
size_t len;
int rw;
{
1998-03-09 03:58:55 +03:00
vm_prot_t prot;
paddr_t pa;
vaddr_t sva, eva;
1998-03-09 03:58:55 +03:00
prot = rw == B_READ ? VM_PROT_READ : VM_PROT_READ|VM_PROT_WRITE;
eva = round_page((vaddr_t)addr + len);
for (sva = trunc_page((vaddr_t)addr); sva < eva; sva += PAGE_SIZE) {
1998-03-09 03:58:55 +03:00
/*
* Extract physical address for the page.
*/
if (pmap_extract(pmap_kernel(), sva, &pa) == FALSE)
1998-03-09 03:58:55 +03:00
panic("chgkprot: invalid page");
pmap_enter(pmap_kernel(), sva, pa, prot, PMAP_WIRED);
1998-03-09 03:58:55 +03:00
}
pmap_update(pmap_kernel());
}
#endif
/*
a whole bunch of changes to improve performance and robustness under load: - remove special treatment of pager_map mappings in pmaps. this is required now, since I've removed the globals that expose the address range. pager_map now uses pmap_kenter_pa() instead of pmap_enter(), so there's no longer any need to special-case it. - eliminate struct uvm_vnode by moving its fields into struct vnode. - rewrite the pageout path. the pager is now responsible for handling the high-level requests instead of only getting control after a bunch of work has already been done on its behalf. this will allow us to UBCify LFS, which needs tighter control over its pages than other filesystems do. writing a page to disk no longer requires making it read-only, which allows us to write wired pages without causing all kinds of havoc. - use a new PG_PAGEOUT flag to indicate that a page should be freed on behalf of the pagedaemon when it's unlocked. this flag is very similar to PG_RELEASED, but unlike PG_RELEASED, PG_PAGEOUT can be cleared if the pageout fails due to eg. an indirect-block buffer being locked. this allows us to remove the "version" field from struct vm_page, and together with shrinking "loan_count" from 32 bits to 16, struct vm_page is now 4 bytes smaller. - no longer use PG_RELEASED for swap-backed pages. if the page is busy because it's being paged out, we can't release the swap slot to be reallocated until that write is complete, but unlike with vnodes we don't keep a count of in-progress writes so there's no good way to know when the write is done. instead, when we need to free a busy swap-backed page, just sleep until we can get it busy ourselves. - implement a fast-path for extending writes which allows us to avoid zeroing new pages. this substantially reduces cpu usage. - encapsulate the data used by the genfs code in a struct genfs_node, which must be the first element of the filesystem-specific vnode data for filesystems which use genfs_{get,put}pages(). - eliminate many of the UVM pagerops, since they aren't needed anymore now that the pager "put" operation is a higher-level operation. - enhance the genfs code to allow NFS to use the genfs_{get,put}pages instead of a modified copy. - clean up struct vnode by removing all the fields that used to be used by the vfs_cluster.c code (which we don't use anymore with UBC). - remove kmem_object and mb_object since they were useless. instead of allocating pages to these objects, we now just allocate pages with no object. such pages are mapped in the kernel until they are freed, so we can use the mapping to find the page to free it. this allows us to remove splvm() protection in several places. The sum of all these changes improves write throughput on my decstation 5000/200 to within 1% of the rate of NetBSD 1.5 and reduces the elapsed time for "make release" of a NetBSD 1.5 source tree on my 128MB pc to 10% less than a 1.5 kernel took.
2001-09-16 00:36:31 +04:00
* uvm_vslock: wire user memory for I/O
*
* - called from physio and sys___sysctl
* - XXXCDC: consider nuking this (or making it a macro?)
*/
int
1999-05-26 05:05:24 +04:00
uvm_vslock(p, addr, len, access_type)
struct proc *p;
1998-03-09 03:58:55 +03:00
caddr_t addr;
size_t len;
1999-05-26 05:05:24 +04:00
vm_prot_t access_type;
{
struct vm_map *map;
vaddr_t start, end;
int error;
map = &p->p_vmspace->vm_map;
start = trunc_page((vaddr_t)addr);
end = round_page((vaddr_t)addr + len);
introduce a new UVM fault type, VM_FAULT_WIREMAX. this is different from VM_FAULT_WIRE in that when the pages being wired are faulted in, the simulated fault is at the maximum protection allowed for the mapping instead of the current protection. use this in uvm_map_pageable{,_all}() to fix the problem where writing via ptrace() to shared libraries that are also mapped with wired mappings in another process causes a diagnostic panic when the wired mapping is removed. this is a really obscure problem so it deserves some more explanation. ptrace() writing to another process ends up down in uvm_map_extract(), which for MAP_PRIVATE mappings (such as shared libraries) will cause the amap to be copied or created. then the amap is made shared (ie. the AMAP_SHARED flag is set) between the kernel and the ptrace()d process so that the kernel can modify pages in the amap and have the ptrace()d process see the changes. then when the page being modified is actually faulted on, the object pages (from the shared library vnode) is copied to a new anon page and inserted into the shared amap. to make all the processes sharing the amap actually see the new anon page instead of the vnode page that was there before, we need to invalidate all the pmap-level mappings of the vnode page in the pmaps of the processes sharing the amap, but we don't have a good way of doing this. the amap doesn't keep track of the vm_maps which map it. so all we can do at this point is to remove all the mappings of the page with pmap_page_protect(), but this has the unfortunate side-effect of removing wired mappings as well. removing wired mappings with pmap_page_protect() is a legitimate operation, it can happen when a file with a wired mapping is truncated. so the pmap has no way of knowing whether a request to remove a wired mapping is normal or when it's due to this weird situation. so the pmap has to remove the weird mapping. the process being ptrace()d goes away and life continues. then, much later when we go to unwire or remove the wired vm_map mapping, we discover that the pmap mapping has been removed when it should still be there, and we panic. so where did we go wrong? the problem is that we don't have any way to update just the pmap mappings that need to be updated in this scenario. we could invent a mechanism to do this, but that is much more complicated than this change and it doesn't seem like the right way to go in the long run either. the real underlying problem here is that wired pmap mappings just aren't a good concept. one of the original properties of the pmap design was supposed to be that all the information in the pmap could be thrown away at any time and the VM system could regenerate it all through fault processing, but wired pmap mappings don't allow that. a better design for UVM would not require wired pmap mappings, and Chuck C. and I are talking about this, but it won't be done anytime soon, so this change will do for now. this change has the effect of causing MAP_PRIVATE mappings to be copied to anonymous memory when they are mlock()d, so that uvm_fault() doesn't need to copy these pages later when called from ptrace(), thus avoiding the call to pmap_page_protect() and the panic that results from this when the mlock()d region is unlocked or freed. note that this change doesn't help the case where the wired mapping is MAP_SHARED. discussed at great length with Chuck Cranor. fixes PRs 10363, 12554, 12604, 13041, 13487, 14580 and 14853.
2002-01-01 01:34:39 +03:00
error = uvm_fault_wire(map, start, end, VM_FAULT_WIRE, access_type);
return error;
}
/*
a whole bunch of changes to improve performance and robustness under load: - remove special treatment of pager_map mappings in pmaps. this is required now, since I've removed the globals that expose the address range. pager_map now uses pmap_kenter_pa() instead of pmap_enter(), so there's no longer any need to special-case it. - eliminate struct uvm_vnode by moving its fields into struct vnode. - rewrite the pageout path. the pager is now responsible for handling the high-level requests instead of only getting control after a bunch of work has already been done on its behalf. this will allow us to UBCify LFS, which needs tighter control over its pages than other filesystems do. writing a page to disk no longer requires making it read-only, which allows us to write wired pages without causing all kinds of havoc. - use a new PG_PAGEOUT flag to indicate that a page should be freed on behalf of the pagedaemon when it's unlocked. this flag is very similar to PG_RELEASED, but unlike PG_RELEASED, PG_PAGEOUT can be cleared if the pageout fails due to eg. an indirect-block buffer being locked. this allows us to remove the "version" field from struct vm_page, and together with shrinking "loan_count" from 32 bits to 16, struct vm_page is now 4 bytes smaller. - no longer use PG_RELEASED for swap-backed pages. if the page is busy because it's being paged out, we can't release the swap slot to be reallocated until that write is complete, but unlike with vnodes we don't keep a count of in-progress writes so there's no good way to know when the write is done. instead, when we need to free a busy swap-backed page, just sleep until we can get it busy ourselves. - implement a fast-path for extending writes which allows us to avoid zeroing new pages. this substantially reduces cpu usage. - encapsulate the data used by the genfs code in a struct genfs_node, which must be the first element of the filesystem-specific vnode data for filesystems which use genfs_{get,put}pages(). - eliminate many of the UVM pagerops, since they aren't needed anymore now that the pager "put" operation is a higher-level operation. - enhance the genfs code to allow NFS to use the genfs_{get,put}pages instead of a modified copy. - clean up struct vnode by removing all the fields that used to be used by the vfs_cluster.c code (which we don't use anymore with UBC). - remove kmem_object and mb_object since they were useless. instead of allocating pages to these objects, we now just allocate pages with no object. such pages are mapped in the kernel until they are freed, so we can use the mapping to find the page to free it. this allows us to remove splvm() protection in several places. The sum of all these changes improves write throughput on my decstation 5000/200 to within 1% of the rate of NetBSD 1.5 and reduces the elapsed time for "make release" of a NetBSD 1.5 source tree on my 128MB pc to 10% less than a 1.5 kernel took.
2001-09-16 00:36:31 +04:00
* uvm_vsunlock: unwire user memory wired by uvm_vslock()
*
* - called from physio and sys___sysctl
* - XXXCDC: consider nuking this (or making it a macro?)
*/
1998-03-09 03:58:55 +03:00
void
uvm_vsunlock(p, addr, len)
struct proc *p;
1998-03-09 03:58:55 +03:00
caddr_t addr;
size_t len;
{
uvm_fault_unwire(&p->p_vmspace->vm_map, trunc_page((vaddr_t)addr),
round_page((vaddr_t)addr + len));
}
/*
* uvm_fork: fork a virtual address space
*
* - the address space is copied as per parent map's inherit values
* - a new "user" structure is allocated for the child process
* [filled in by MD layer...]
* - if specified, the child gets a new user stack described by
* stack and stacksize
* - NOTE: the kernel stack may be at a different location in the child
* process, and thus addresses of automatic variables may be invalid
* after cpu_fork returns in the child process. We do nothing here
* after cpu_fork returns.
* - XXXCDC: we need a way for this to return a failure value rather
* than just hang
*/
1998-03-09 03:58:55 +03:00
void
uvm_fork(p1, p2, shared, stack, stacksize, func, arg)
1998-03-09 03:58:55 +03:00
struct proc *p1, *p2;
boolean_t shared;
void *stack;
size_t stacksize;
void (*func) __P((void *));
void *arg;
{
struct user *up = p2->p_addr;
int error;
if (shared == TRUE) {
p2->p_vmspace = NULL;
a whole bunch of changes to improve performance and robustness under load: - remove special treatment of pager_map mappings in pmaps. this is required now, since I've removed the globals that expose the address range. pager_map now uses pmap_kenter_pa() instead of pmap_enter(), so there's no longer any need to special-case it. - eliminate struct uvm_vnode by moving its fields into struct vnode. - rewrite the pageout path. the pager is now responsible for handling the high-level requests instead of only getting control after a bunch of work has already been done on its behalf. this will allow us to UBCify LFS, which needs tighter control over its pages than other filesystems do. writing a page to disk no longer requires making it read-only, which allows us to write wired pages without causing all kinds of havoc. - use a new PG_PAGEOUT flag to indicate that a page should be freed on behalf of the pagedaemon when it's unlocked. this flag is very similar to PG_RELEASED, but unlike PG_RELEASED, PG_PAGEOUT can be cleared if the pageout fails due to eg. an indirect-block buffer being locked. this allows us to remove the "version" field from struct vm_page, and together with shrinking "loan_count" from 32 bits to 16, struct vm_page is now 4 bytes smaller. - no longer use PG_RELEASED for swap-backed pages. if the page is busy because it's being paged out, we can't release the swap slot to be reallocated until that write is complete, but unlike with vnodes we don't keep a count of in-progress writes so there's no good way to know when the write is done. instead, when we need to free a busy swap-backed page, just sleep until we can get it busy ourselves. - implement a fast-path for extending writes which allows us to avoid zeroing new pages. this substantially reduces cpu usage. - encapsulate the data used by the genfs code in a struct genfs_node, which must be the first element of the filesystem-specific vnode data for filesystems which use genfs_{get,put}pages(). - eliminate many of the UVM pagerops, since they aren't needed anymore now that the pager "put" operation is a higher-level operation. - enhance the genfs code to allow NFS to use the genfs_{get,put}pages instead of a modified copy. - clean up struct vnode by removing all the fields that used to be used by the vfs_cluster.c code (which we don't use anymore with UBC). - remove kmem_object and mb_object since they were useless. instead of allocating pages to these objects, we now just allocate pages with no object. such pages are mapped in the kernel until they are freed, so we can use the mapping to find the page to free it. this allows us to remove splvm() protection in several places. The sum of all these changes improves write throughput on my decstation 5000/200 to within 1% of the rate of NetBSD 1.5 and reduces the elapsed time for "make release" of a NetBSD 1.5 source tree on my 128MB pc to 10% less than a 1.5 kernel took.
2001-09-16 00:36:31 +04:00
uvmspace_share(p1, p2);
} else
a whole bunch of changes to improve performance and robustness under load: - remove special treatment of pager_map mappings in pmaps. this is required now, since I've removed the globals that expose the address range. pager_map now uses pmap_kenter_pa() instead of pmap_enter(), so there's no longer any need to special-case it. - eliminate struct uvm_vnode by moving its fields into struct vnode. - rewrite the pageout path. the pager is now responsible for handling the high-level requests instead of only getting control after a bunch of work has already been done on its behalf. this will allow us to UBCify LFS, which needs tighter control over its pages than other filesystems do. writing a page to disk no longer requires making it read-only, which allows us to write wired pages without causing all kinds of havoc. - use a new PG_PAGEOUT flag to indicate that a page should be freed on behalf of the pagedaemon when it's unlocked. this flag is very similar to PG_RELEASED, but unlike PG_RELEASED, PG_PAGEOUT can be cleared if the pageout fails due to eg. an indirect-block buffer being locked. this allows us to remove the "version" field from struct vm_page, and together with shrinking "loan_count" from 32 bits to 16, struct vm_page is now 4 bytes smaller. - no longer use PG_RELEASED for swap-backed pages. if the page is busy because it's being paged out, we can't release the swap slot to be reallocated until that write is complete, but unlike with vnodes we don't keep a count of in-progress writes so there's no good way to know when the write is done. instead, when we need to free a busy swap-backed page, just sleep until we can get it busy ourselves. - implement a fast-path for extending writes which allows us to avoid zeroing new pages. this substantially reduces cpu usage. - encapsulate the data used by the genfs code in a struct genfs_node, which must be the first element of the filesystem-specific vnode data for filesystems which use genfs_{get,put}pages(). - eliminate many of the UVM pagerops, since they aren't needed anymore now that the pager "put" operation is a higher-level operation. - enhance the genfs code to allow NFS to use the genfs_{get,put}pages instead of a modified copy. - clean up struct vnode by removing all the fields that used to be used by the vfs_cluster.c code (which we don't use anymore with UBC). - remove kmem_object and mb_object since they were useless. instead of allocating pages to these objects, we now just allocate pages with no object. such pages are mapped in the kernel until they are freed, so we can use the mapping to find the page to free it. this allows us to remove splvm() protection in several places. The sum of all these changes improves write throughput on my decstation 5000/200 to within 1% of the rate of NetBSD 1.5 and reduces the elapsed time for "make release" of a NetBSD 1.5 source tree on my 128MB pc to 10% less than a 1.5 kernel took.
2001-09-16 00:36:31 +04:00
p2->p_vmspace = uvmspace_fork(p1->p_vmspace);
1998-03-09 03:58:55 +03:00
/*
* Wire down the U-area for the process, which contains the PCB
* and the kernel stack. Wired state is stored in p->p_flag's
* P_INMEM bit rather than in the vm_map_entry's wired count
* to prevent kernel_map fragmentation.
*
* Note the kernel stack gets read/write accesses right off
* the bat.
1998-03-09 03:58:55 +03:00
*/
introduce a new UVM fault type, VM_FAULT_WIREMAX. this is different from VM_FAULT_WIRE in that when the pages being wired are faulted in, the simulated fault is at the maximum protection allowed for the mapping instead of the current protection. use this in uvm_map_pageable{,_all}() to fix the problem where writing via ptrace() to shared libraries that are also mapped with wired mappings in another process causes a diagnostic panic when the wired mapping is removed. this is a really obscure problem so it deserves some more explanation. ptrace() writing to another process ends up down in uvm_map_extract(), which for MAP_PRIVATE mappings (such as shared libraries) will cause the amap to be copied or created. then the amap is made shared (ie. the AMAP_SHARED flag is set) between the kernel and the ptrace()d process so that the kernel can modify pages in the amap and have the ptrace()d process see the changes. then when the page being modified is actually faulted on, the object pages (from the shared library vnode) is copied to a new anon page and inserted into the shared amap. to make all the processes sharing the amap actually see the new anon page instead of the vnode page that was there before, we need to invalidate all the pmap-level mappings of the vnode page in the pmaps of the processes sharing the amap, but we don't have a good way of doing this. the amap doesn't keep track of the vm_maps which map it. so all we can do at this point is to remove all the mappings of the page with pmap_page_protect(), but this has the unfortunate side-effect of removing wired mappings as well. removing wired mappings with pmap_page_protect() is a legitimate operation, it can happen when a file with a wired mapping is truncated. so the pmap has no way of knowing whether a request to remove a wired mapping is normal or when it's due to this weird situation. so the pmap has to remove the weird mapping. the process being ptrace()d goes away and life continues. then, much later when we go to unwire or remove the wired vm_map mapping, we discover that the pmap mapping has been removed when it should still be there, and we panic. so where did we go wrong? the problem is that we don't have any way to update just the pmap mappings that need to be updated in this scenario. we could invent a mechanism to do this, but that is much more complicated than this change and it doesn't seem like the right way to go in the long run either. the real underlying problem here is that wired pmap mappings just aren't a good concept. one of the original properties of the pmap design was supposed to be that all the information in the pmap could be thrown away at any time and the VM system could regenerate it all through fault processing, but wired pmap mappings don't allow that. a better design for UVM would not require wired pmap mappings, and Chuck C. and I are talking about this, but it won't be done anytime soon, so this change will do for now. this change has the effect of causing MAP_PRIVATE mappings to be copied to anonymous memory when they are mlock()d, so that uvm_fault() doesn't need to copy these pages later when called from ptrace(), thus avoiding the call to pmap_page_protect() and the panic that results from this when the mlock()d region is unlocked or freed. note that this change doesn't help the case where the wired mapping is MAP_SHARED. discussed at great length with Chuck Cranor. fixes PRs 10363, 12554, 12604, 13041, 13487, 14580 and 14853.
2002-01-01 01:34:39 +03:00
error = uvm_fault_wire(kernel_map, (vaddr_t)up, (vaddr_t)up + USPACE,
VM_FAULT_WIRE, VM_PROT_READ | VM_PROT_WRITE);
if (error)
panic("uvm_fork: uvm_fault_wire failed: %d", error);
1998-03-09 03:58:55 +03:00
#ifdef KSTACK_CHECK_MAGIC
/*
* fill stack with magic number
*/
kstack_setup_magic(p2);
#endif
1998-03-09 03:58:55 +03:00
/*
* p_stats currently points at a field in the user struct. Copy
* parts of p_stats, and zero out the rest.
1998-03-09 03:58:55 +03:00
*/
p2->p_stats = &up->u_stats;
memset(&up->u_stats.pstat_startzero, 0,
((caddr_t)&up->u_stats.pstat_endzero -
(caddr_t)&up->u_stats.pstat_startzero));
memcpy(&up->u_stats.pstat_startcopy, &p1->p_stats->pstat_startcopy,
((caddr_t)&up->u_stats.pstat_endcopy -
(caddr_t)&up->u_stats.pstat_startcopy));
2001-05-25 08:06:11 +04:00
1998-03-09 03:58:55 +03:00
/*
* cpu_fork() copy and update the pcb, and make the child ready
* to run. If this is a normal user fork, the child will exit
* directly to user mode via child_return() on its first time
* slice and will not return here. If this is a kernel thread,
* the specified entry point will be executed.
1998-03-09 03:58:55 +03:00
*/
cpu_fork(p1, p2, stack, stacksize, func, arg);
}
/*
* uvm_exit: exit a virtual address space
*
* - the process passed to us is a dead (pre-zombie) process; we
* are running on a different context now (the reaper).
* - we must run in a separate thread because freeing the vmspace
* of the dead process may block.
*/
void
uvm_exit(p)
struct proc *p;
{
vaddr_t va = (vaddr_t)p->p_addr;
uvmspace_free(p->p_vmspace);
p->p_flag &= ~P_INMEM;
uvm_km_free(kernel_map, va, USPACE);
p->p_addr = NULL;
}
/*
* uvm_init_limit: init per-process VM limits
*
* - called for process 0 and then inherited by all others.
*/
1998-03-09 03:58:55 +03:00
void
uvm_init_limits(p)
struct proc *p;
{
1998-03-09 03:58:55 +03:00
/*
* Set up the initial limits on process VM. Set the maximum
* resident set size to be all of (reasonably) available memory.
* This causes any single, large process to start random page
* replacement once it fills memory.
*/
p->p_rlimit[RLIMIT_STACK].rlim_cur = DFLSSIZ;
p->p_rlimit[RLIMIT_STACK].rlim_max = MAXSSIZ;
p->p_rlimit[RLIMIT_DATA].rlim_cur = DFLDSIZ;
p->p_rlimit[RLIMIT_DATA].rlim_max = MAXDSIZ;
p->p_rlimit[RLIMIT_RSS].rlim_cur = ptoa(uvmexp.free);
}
#ifdef DEBUG
int enableswap = 1;
int swapdebug = 0;
#define SDB_FOLLOW 1
#define SDB_SWAPIN 2
#define SDB_SWAPOUT 4
#endif
/*
* uvm_swapin: swap in a process's u-area.
*/
1998-03-09 03:58:55 +03:00
void
uvm_swapin(p)
struct proc *p;
{
vaddr_t addr;
a whole bunch of changes to improve performance and robustness under load: - remove special treatment of pager_map mappings in pmaps. this is required now, since I've removed the globals that expose the address range. pager_map now uses pmap_kenter_pa() instead of pmap_enter(), so there's no longer any need to special-case it. - eliminate struct uvm_vnode by moving its fields into struct vnode. - rewrite the pageout path. the pager is now responsible for handling the high-level requests instead of only getting control after a bunch of work has already been done on its behalf. this will allow us to UBCify LFS, which needs tighter control over its pages than other filesystems do. writing a page to disk no longer requires making it read-only, which allows us to write wired pages without causing all kinds of havoc. - use a new PG_PAGEOUT flag to indicate that a page should be freed on behalf of the pagedaemon when it's unlocked. this flag is very similar to PG_RELEASED, but unlike PG_RELEASED, PG_PAGEOUT can be cleared if the pageout fails due to eg. an indirect-block buffer being locked. this allows us to remove the "version" field from struct vm_page, and together with shrinking "loan_count" from 32 bits to 16, struct vm_page is now 4 bytes smaller. - no longer use PG_RELEASED for swap-backed pages. if the page is busy because it's being paged out, we can't release the swap slot to be reallocated until that write is complete, but unlike with vnodes we don't keep a count of in-progress writes so there's no good way to know when the write is done. instead, when we need to free a busy swap-backed page, just sleep until we can get it busy ourselves. - implement a fast-path for extending writes which allows us to avoid zeroing new pages. this substantially reduces cpu usage. - encapsulate the data used by the genfs code in a struct genfs_node, which must be the first element of the filesystem-specific vnode data for filesystems which use genfs_{get,put}pages(). - eliminate many of the UVM pagerops, since they aren't needed anymore now that the pager "put" operation is a higher-level operation. - enhance the genfs code to allow NFS to use the genfs_{get,put}pages instead of a modified copy. - clean up struct vnode by removing all the fields that used to be used by the vfs_cluster.c code (which we don't use anymore with UBC). - remove kmem_object and mb_object since they were useless. instead of allocating pages to these objects, we now just allocate pages with no object. such pages are mapped in the kernel until they are freed, so we can use the mapping to find the page to free it. this allows us to remove splvm() protection in several places. The sum of all these changes improves write throughput on my decstation 5000/200 to within 1% of the rate of NetBSD 1.5 and reduces the elapsed time for "make release" of a NetBSD 1.5 source tree on my 128MB pc to 10% less than a 1.5 kernel took.
2001-09-16 00:36:31 +04:00
int s, error;
1998-03-09 03:58:55 +03:00
addr = (vaddr_t)p->p_addr;
1998-03-09 03:58:55 +03:00
/* make P_INMEM true */
introduce a new UVM fault type, VM_FAULT_WIREMAX. this is different from VM_FAULT_WIRE in that when the pages being wired are faulted in, the simulated fault is at the maximum protection allowed for the mapping instead of the current protection. use this in uvm_map_pageable{,_all}() to fix the problem where writing via ptrace() to shared libraries that are also mapped with wired mappings in another process causes a diagnostic panic when the wired mapping is removed. this is a really obscure problem so it deserves some more explanation. ptrace() writing to another process ends up down in uvm_map_extract(), which for MAP_PRIVATE mappings (such as shared libraries) will cause the amap to be copied or created. then the amap is made shared (ie. the AMAP_SHARED flag is set) between the kernel and the ptrace()d process so that the kernel can modify pages in the amap and have the ptrace()d process see the changes. then when the page being modified is actually faulted on, the object pages (from the shared library vnode) is copied to a new anon page and inserted into the shared amap. to make all the processes sharing the amap actually see the new anon page instead of the vnode page that was there before, we need to invalidate all the pmap-level mappings of the vnode page in the pmaps of the processes sharing the amap, but we don't have a good way of doing this. the amap doesn't keep track of the vm_maps which map it. so all we can do at this point is to remove all the mappings of the page with pmap_page_protect(), but this has the unfortunate side-effect of removing wired mappings as well. removing wired mappings with pmap_page_protect() is a legitimate operation, it can happen when a file with a wired mapping is truncated. so the pmap has no way of knowing whether a request to remove a wired mapping is normal or when it's due to this weird situation. so the pmap has to remove the weird mapping. the process being ptrace()d goes away and life continues. then, much later when we go to unwire or remove the wired vm_map mapping, we discover that the pmap mapping has been removed when it should still be there, and we panic. so where did we go wrong? the problem is that we don't have any way to update just the pmap mappings that need to be updated in this scenario. we could invent a mechanism to do this, but that is much more complicated than this change and it doesn't seem like the right way to go in the long run either. the real underlying problem here is that wired pmap mappings just aren't a good concept. one of the original properties of the pmap design was supposed to be that all the information in the pmap could be thrown away at any time and the VM system could regenerate it all through fault processing, but wired pmap mappings don't allow that. a better design for UVM would not require wired pmap mappings, and Chuck C. and I are talking about this, but it won't be done anytime soon, so this change will do for now. this change has the effect of causing MAP_PRIVATE mappings to be copied to anonymous memory when they are mlock()d, so that uvm_fault() doesn't need to copy these pages later when called from ptrace(), thus avoiding the call to pmap_page_protect() and the panic that results from this when the mlock()d region is unlocked or freed. note that this change doesn't help the case where the wired mapping is MAP_SHARED. discussed at great length with Chuck Cranor. fixes PRs 10363, 12554, 12604, 13041, 13487, 14580 and 14853.
2002-01-01 01:34:39 +03:00
error = uvm_fault_wire(kernel_map, addr, addr + USPACE, VM_FAULT_WIRE,
VM_PROT_READ | VM_PROT_WRITE);
a whole bunch of changes to improve performance and robustness under load: - remove special treatment of pager_map mappings in pmaps. this is required now, since I've removed the globals that expose the address range. pager_map now uses pmap_kenter_pa() instead of pmap_enter(), so there's no longer any need to special-case it. - eliminate struct uvm_vnode by moving its fields into struct vnode. - rewrite the pageout path. the pager is now responsible for handling the high-level requests instead of only getting control after a bunch of work has already been done on its behalf. this will allow us to UBCify LFS, which needs tighter control over its pages than other filesystems do. writing a page to disk no longer requires making it read-only, which allows us to write wired pages without causing all kinds of havoc. - use a new PG_PAGEOUT flag to indicate that a page should be freed on behalf of the pagedaemon when it's unlocked. this flag is very similar to PG_RELEASED, but unlike PG_RELEASED, PG_PAGEOUT can be cleared if the pageout fails due to eg. an indirect-block buffer being locked. this allows us to remove the "version" field from struct vm_page, and together with shrinking "loan_count" from 32 bits to 16, struct vm_page is now 4 bytes smaller. - no longer use PG_RELEASED for swap-backed pages. if the page is busy because it's being paged out, we can't release the swap slot to be reallocated until that write is complete, but unlike with vnodes we don't keep a count of in-progress writes so there's no good way to know when the write is done. instead, when we need to free a busy swap-backed page, just sleep until we can get it busy ourselves. - implement a fast-path for extending writes which allows us to avoid zeroing new pages. this substantially reduces cpu usage. - encapsulate the data used by the genfs code in a struct genfs_node, which must be the first element of the filesystem-specific vnode data for filesystems which use genfs_{get,put}pages(). - eliminate many of the UVM pagerops, since they aren't needed anymore now that the pager "put" operation is a higher-level operation. - enhance the genfs code to allow NFS to use the genfs_{get,put}pages instead of a modified copy. - clean up struct vnode by removing all the fields that used to be used by the vfs_cluster.c code (which we don't use anymore with UBC). - remove kmem_object and mb_object since they were useless. instead of allocating pages to these objects, we now just allocate pages with no object. such pages are mapped in the kernel until they are freed, so we can use the mapping to find the page to free it. this allows us to remove splvm() protection in several places. The sum of all these changes improves write throughput on my decstation 5000/200 to within 1% of the rate of NetBSD 1.5 and reduces the elapsed time for "make release" of a NetBSD 1.5 source tree on my 128MB pc to 10% less than a 1.5 kernel took.
2001-09-16 00:36:31 +04:00
if (error) {
panic("uvm_swapin: rewiring stack failed: %d", error);
}
1998-03-09 03:58:55 +03:00
/*
* Some architectures need to be notified when the user area has
* moved to new physical page(s) (e.g. see mips/mips/vm_machdep.c).
*/
cpu_swapin(p);
SCHED_LOCK(s);
1998-03-09 03:58:55 +03:00
if (p->p_stat == SRUN)
setrunqueue(p);
p->p_flag |= P_INMEM;
SCHED_UNLOCK(s);
1998-03-09 03:58:55 +03:00
p->p_swtime = 0;
++uvmexp.swapins;
}
/*
* uvm_scheduler: process zero main loop
*
* - attempt to swapin every swaped-out, runnable process in order of
* priority.
* - if not enough memory, wake the pagedaemon and let it clear space.
*/
1998-03-09 03:58:55 +03:00
void
uvm_scheduler()
{
2000-03-30 16:31:50 +04:00
struct proc *p;
int pri;
1998-03-09 03:58:55 +03:00
struct proc *pp;
int ppri;
loop:
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
while (!enableswap)
tsleep(&proc0, PVM, "noswap", 0);
#endif
1998-03-09 03:58:55 +03:00
pp = NULL; /* process to choose */
ppri = INT_MIN; /* its priority */
proclist_lock_read();
LIST_FOREACH(p, &allproc, p_list) {
1998-03-09 03:58:55 +03:00
/* is it a runnable swapped out process? */
if (p->p_stat == SRUN && (p->p_flag & P_INMEM) == 0) {
pri = p->p_swtime + p->p_slptime -
(p->p_nice - NZERO) * 8;
if (pri > ppri) { /* higher priority? remember it. */
pp = p;
ppri = pri;
}
}
}
/*
* XXXSMP: possible unlock/sleep race between here and the
* "scheduler" tsleep below..
*/
proclist_unlock_read();
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (swapdebug & SDB_FOLLOW)
printf("scheduler: running, procp %p pri %d\n", pp, ppri);
#endif
1998-03-09 03:58:55 +03:00
/*
* Nothing to do, back to sleep
*/
if ((p = pp) == NULL) {
tsleep(&proc0, PVM, "scheduler", 0);
1998-03-09 03:58:55 +03:00
goto loop;
}
/*
* we have found swapped out process which we would like to bring
* back in.
*
* XXX: this part is really bogus cuz we could deadlock on memory
* despite our feeble check
*/
if (uvmexp.free > atop(USPACE)) {
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (swapdebug & SDB_SWAPIN)
printf("swapin: pid %d(%s)@%p, pri %d free %d\n",
p->p_pid, p->p_comm, p->p_addr, ppri, uvmexp.free);
#endif
1998-03-09 03:58:55 +03:00
uvm_swapin(p);
goto loop;
}
/*
* not enough memory, jab the pageout daemon and wait til the coast
* is clear
*/
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (swapdebug & SDB_FOLLOW)
printf("scheduler: no room for pid %d(%s), free %d\n",
p->p_pid, p->p_comm, uvmexp.free);
#endif
1998-03-09 03:58:55 +03:00
uvm_wait("schedpwait");
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (swapdebug & SDB_FOLLOW)
printf("scheduler: room again, free %d\n", uvmexp.free);
#endif
1998-03-09 03:58:55 +03:00
goto loop;
}
/*
* swappable: is process "p" swappable?
*/
#define swappable(p) \
(((p)->p_flag & (P_SYSTEM | P_INMEM | P_WEXIT)) == P_INMEM && \
(p)->p_holdcnt == 0)
/*
* swapout_threads: find threads that can be swapped and unwire their
* u-areas.
*
* - called by the pagedaemon
* - try and swap at least one processs
* - processes that are sleeping or stopped for maxslp or more seconds
* are swapped... otherwise the longest-sleeping or stopped process
* is swapped, otherwise the longest resident process...
*/
1998-03-09 03:58:55 +03:00
void
uvm_swapout_threads()
{
2000-03-30 16:31:50 +04:00
struct proc *p;
1998-03-09 03:58:55 +03:00
struct proc *outp, *outp2;
int outpri, outpri2;
int didswap = 0;
2001-05-25 08:06:11 +04:00
extern int maxslp;
1998-03-09 03:58:55 +03:00
/* XXXCDC: should move off to uvmexp. or uvm., also in uvm_meter */
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (!enableswap)
return;
#endif
1998-03-09 03:58:55 +03:00
/*
* outp/outpri : stop/sleep process with largest sleeptime < maxslp
* outp2/outpri2: the longest resident process (its swap time)
*/
outp = outp2 = NULL;
outpri = outpri2 = 0;
proclist_lock_read();
LIST_FOREACH(p, &allproc, p_list) {
1998-03-09 03:58:55 +03:00
if (!swappable(p))
continue;
switch (p->p_stat) {
case SRUN:
case SONPROC:
1998-03-09 03:58:55 +03:00
if (p->p_swtime > outpri2) {
outp2 = p;
outpri2 = p->p_swtime;
}
continue;
2001-05-25 08:06:11 +04:00
1998-03-09 03:58:55 +03:00
case SSLEEP:
case SSTOP:
if (p->p_slptime >= maxslp) {
uvm_swapout(p);
1998-03-09 03:58:55 +03:00
didswap++;
} else if (p->p_slptime > outpri) {
outp = p;
outpri = p->p_slptime;
}
continue;
}
}
proclist_unlock_read();
1998-03-09 03:58:55 +03:00
/*
* If we didn't get rid of any real duds, toss out the next most
* likely sleeping/stopped or running candidate. We only do this
* if we are real low on memory since we don't gain much by doing
* it (USPACE bytes).
*/
if (didswap == 0 && uvmexp.free <= atop(round_page(USPACE))) {
if ((p = outp) == NULL)
p = outp2;
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (swapdebug & SDB_SWAPOUT)
printf("swapout_threads: no duds, try procp %p\n", p);
#endif
1998-03-09 03:58:55 +03:00
if (p)
uvm_swapout(p);
}
}
/*
* uvm_swapout: swap out process "p"
*
2001-05-25 08:06:11 +04:00
* - currently "swapout" means "unwire U-area" and "pmap_collect()"
* the pmap.
* - XXXCDC: should deactivate all process' private anonymous memory
*/
1998-03-09 03:58:55 +03:00
static void
uvm_swapout(p)
2000-03-30 16:31:50 +04:00
struct proc *p;
{
vaddr_t addr;
1998-03-09 03:58:55 +03:00
int s;
#ifdef DEBUG
1998-03-09 03:58:55 +03:00
if (swapdebug & SDB_SWAPOUT)
printf("swapout: pid %d(%s)@%p, stat %x pri %d free %d\n",
p->p_pid, p->p_comm, p->p_addr, p->p_stat,
p->p_slptime, uvmexp.free);
#endif
1998-03-09 03:58:55 +03:00
/*
* Do any machine-specific actions necessary before swapout.
* This can include saving floating point state, etc.
*/
cpu_swapout(p);
/*
* Mark it as (potentially) swapped out.
*/
SCHED_LOCK(s);
1998-03-09 03:58:55 +03:00
p->p_flag &= ~P_INMEM;
if (p->p_stat == SRUN)
remrunqueue(p);
SCHED_UNLOCK(s);
1998-03-09 03:58:55 +03:00
p->p_swtime = 0;
p->p_stats->p_ru.ru_nswap++;
1998-03-09 03:58:55 +03:00
++uvmexp.swapouts;
/*
* Unwire the to-be-swapped process's user struct and kernel stack.
*/
addr = (vaddr_t)p->p_addr;
uvm_fault_unwire(kernel_map, addr, addr + USPACE); /* !P_INMEM */
pmap_collect(vm_map_pmap(&p->p_vmspace->vm_map));
}
/*
* uvm_coredump_walkmap: walk a process's map for the purpose of dumping
* a core file.
*/
int
uvm_coredump_walkmap(p, vp, cred, func, cookie)
struct proc *p;
struct vnode *vp;
struct ucred *cred;
int (*func)(struct proc *, struct vnode *, struct ucred *,
struct uvm_coredump_state *);
void *cookie;
{
struct uvm_coredump_state state;
struct vmspace *vm = p->p_vmspace;
struct vm_map *map = &vm->vm_map;
struct vm_map_entry *entry;
vaddr_t maxstack;
int error;
maxstack = trunc_page(USRSTACK - ctob(vm->vm_ssize));
for (entry = map->header.next; entry != &map->header;
entry = entry->next) {
/* Should never happen for a user process. */
if (UVM_ET_ISSUBMAP(entry))
panic("uvm_coredump_walkmap: user process with "
"submap?");
state.cookie = cookie;
state.start = entry->start;
state.end = entry->end;
state.prot = entry->protection;
state.flags = 0;
if (state.start >= VM_MAXUSER_ADDRESS)
continue;
if (state.end > VM_MAXUSER_ADDRESS)
state.end = VM_MAXUSER_ADDRESS;
if (state.start >= (vaddr_t)vm->vm_maxsaddr) {
if (state.end <= maxstack)
continue;
if (state.start < maxstack)
state.start = maxstack;
state.flags |= UVM_COREDUMP_STACK;
}
if ((entry->protection & VM_PROT_WRITE) == 0)
state.flags |= UVM_COREDUMP_NODUMP;
if (entry->object.uvm_obj != NULL &&
entry->object.uvm_obj->pgops == &uvm_deviceops)
state.flags |= UVM_COREDUMP_NODUMP;
error = (*func)(p, vp, cred, &state);
if (error)
return (error);
}
return (0);
}