From 19b9de9868fdc27a19baf65bf6292570a670de7e Mon Sep 17 00:00:00 2001 From: yamt Date: Fri, 29 Feb 2008 20:35:23 +0000 Subject: [PATCH] uvm_swap_io: if pagedaemon, don't wait for iobuf. --- sys/uvm/uvm_extern.h | 4 ++- sys/uvm/uvm_pager.c | 69 +++++++++++++++++++++++++------------------ sys/uvm/uvm_pdaemon.c | 6 ++-- sys/uvm/uvm_swap.c | 21 ++++++++----- 4 files changed, 60 insertions(+), 40 deletions(-) diff --git a/sys/uvm/uvm_extern.h b/sys/uvm/uvm_extern.h index 5566c36ecb68..2e9464381e7a 100644 --- a/sys/uvm/uvm_extern.h +++ b/sys/uvm/uvm_extern.h @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_extern.h,v 1.144 2008/01/28 12:22:47 yamt Exp $ */ +/* $NetBSD: uvm_extern.h,v 1.145 2008/02/29 20:35:23 yamt Exp $ */ /* * @@ -690,6 +690,8 @@ void uvm_setpagesize(void); void uvm_aio_biodone1(struct buf *); void uvm_aio_biodone(struct buf *); void uvm_aio_aiodone(struct buf *); +void uvm_aio_aiodone_pages(struct vm_page **, int, bool, + int); /* uvm_pdaemon.c */ void uvm_pageout(void *); diff --git a/sys/uvm/uvm_pager.c b/sys/uvm/uvm_pager.c index eb5d2b9ee3ac..c1efccff17f8 100644 --- a/sys/uvm/uvm_pager.c +++ b/sys/uvm/uvm_pager.c @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_pager.c,v 1.90 2008/01/02 11:49:19 ad Exp $ */ +/* $NetBSD: uvm_pager.c,v 1.91 2008/02/29 20:35:23 yamt Exp $ */ /* * @@ -39,7 +39,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.90 2008/01/02 11:49:19 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.91 2008/02/29 20:35:23 yamt Exp $"); #include "opt_uvmhist.h" #include "opt_readahead.h" @@ -291,39 +291,21 @@ uvm_aio_biodone(struct buf *bp) workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL); } -/* - * uvm_aio_aiodone: do iodone processing for async i/os. - * this should be called in thread context, not interrupt context. - */ - void -uvm_aio_aiodone(struct buf *bp) +uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error) { - int npages = bp->b_bufsize >> PAGE_SHIFT; - struct vm_page *pg, *pgs[npages]; struct uvm_object *uobj; + struct vm_page *pg; kmutex_t *slock; - int i, error, swslot; - int pageout_done = 0; - bool write, swap; - UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); - UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); - - error = bp->b_error; - write = (bp->b_flags & B_READ) == 0; - /* XXXUBC BC_NOCACHE is for swap pager, should be done differently */ - if (write && !(bp->b_cflags & BC_NOCACHE) && bioopsp != NULL) - (*bioopsp->io_pageiodone)(bp); - - uobj = NULL; - for (i = 0; i < npages; i++) { - pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); - UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); - } - uvm_pagermapout((vaddr_t)bp->b_data, npages); + int pageout_done; + int swslot; + int i; + bool swap; swslot = 0; + pageout_done = 0; slock = NULL; + uobj = NULL; pg = pgs[0]; swap = (pg->uanon != NULL && pg->uobject == NULL) || (pg->pqflags & PQ_AOBJ) != 0; @@ -484,6 +466,37 @@ uvm_aio_aiodone(struct buf *bp) uvmexp.pdpending--; #endif /* defined(VMSWAP) */ } +} + +/* + * uvm_aio_aiodone: do iodone processing for async i/os. + * this should be called in thread context, not interrupt context. + */ + +void +uvm_aio_aiodone(struct buf *bp) +{ + int npages = bp->b_bufsize >> PAGE_SHIFT; + struct vm_page *pgs[npages]; + int i, error; + bool write; + UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); + UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); + + error = bp->b_error; + write = (bp->b_flags & B_READ) == 0; + /* XXXUBC BC_NOCACHE is for swap pager, should be done differently */ + if (write && !(bp->b_cflags & BC_NOCACHE) && bioopsp != NULL) + (*bioopsp->io_pageiodone)(bp); + + for (i = 0; i < npages; i++) { + pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); + UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); + } + uvm_pagermapout((vaddr_t)bp->b_data, npages); + + uvm_aio_aiodone_pages(pgs, npages, write, error); + if (write && (bp->b_cflags & BC_AGE) != 0) { mutex_enter(bp->b_objlock); vwakeup(bp); diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c index 38ee3cc720aa..c560739651a9 100644 --- a/sys/uvm/uvm_pdaemon.c +++ b/sys/uvm/uvm_pdaemon.c @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_pdaemon.c,v 1.91 2008/02/07 12:24:16 yamt Exp $ */ +/* $NetBSD: uvm_pdaemon.c,v 1.92 2008/02/29 20:35:23 yamt Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -71,7 +71,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.91 2008/02/07 12:24:16 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.92 2008/02/29 20:35:23 yamt Exp $"); #include "opt_uvmhist.h" #include "opt_readahead.h" @@ -537,7 +537,7 @@ swapcluster_flush(struct swapcluster *swc, bool now) uvmexp.pdpageouts++; uvm_pageout_start(nused); error = uvm_swap_put(slot, swc->swc_pages, nused, 0); - KASSERT(error == 0); + KASSERT(error == 0 || error == ENOMEM); } /* diff --git a/sys/uvm/uvm_swap.c b/sys/uvm/uvm_swap.c index 508f80b29214..0872224221eb 100644 --- a/sys/uvm/uvm_swap.c +++ b/sys/uvm/uvm_swap.c @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_swap.c,v 1.136 2008/01/30 14:25:21 hannken Exp $ */ +/* $NetBSD: uvm_swap.c,v 1.137 2008/02/29 20:35:23 yamt Exp $ */ /* * Copyright (c) 1995, 1996, 1997 Matthew R. Green @@ -32,7 +32,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.136 2008/01/30 14:25:21 hannken Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.137 2008/02/29 20:35:23 yamt Exp $"); #include "fs_nfs.h" #include "opt_uvmhist.h" @@ -1699,6 +1699,17 @@ uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) write = (flags & B_READ) == 0; async = (flags & B_ASYNC) != 0; + /* + * allocate a buf for the i/o. + */ + + KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async)); + bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp); + if (bp == NULL) { + uvm_aio_aiodone_pages(pps, npages, true, ENOMEM); + return ENOMEM; + } + /* * convert starting drum slot to block number */ @@ -1714,12 +1725,6 @@ uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; kva = uvm_pagermapin(pps, npages, mapinflags); - /* - * now allocate a buf for the i/o. - */ - - bp = getiobuf(swapdev_vp, true); - /* * fill in the bp/sbp. we currently route our i/o through * /dev/drum's vnode [swapdev_vp].