uvm_swap_io: if pagedaemon, don't wait for iobuf.

sys/uvm/uvm_extern.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_extern.h,v 1.144 2008/01/28 12:22:47 yamt Exp $	*/
+/*	$NetBSD: uvm_extern.h,v 1.145 2008/02/29 20:35:23 yamt Exp $	*/
 
 /*
  *
@@ -690,6 +690,8 @@ void			uvm_setpagesize(void);
 void			uvm_aio_biodone1(struct buf *);
 void			uvm_aio_biodone(struct buf *);
 void			uvm_aio_aiodone(struct buf *);
+void			uvm_aio_aiodone_pages(struct vm_page **, int, bool,
+			    int);
 
 /* uvm_pdaemon.c */
 void			uvm_pageout(void *);

sys/uvm/uvm_pager.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_pager.c,v 1.90 2008/01/02 11:49:19 ad Exp $	*/
+/*	$NetBSD: uvm_pager.c,v 1.91 2008/02/29 20:35:23 yamt Exp $	*/
 
 /*
  *
@@ -39,7 +39,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.90 2008/01/02 11:49:19 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.91 2008/02/29 20:35:23 yamt Exp $");
 
 #include "opt_uvmhist.h"
 #include "opt_readahead.h"
@@ -291,39 +291,21 @@ uvm_aio_biodone(struct buf *bp)
 	workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL);
 }
 
-/*
- * uvm_aio_aiodone: do iodone processing for async i/os.
- * this should be called in thread context, not interrupt context.
- */
-
 void
-uvm_aio_aiodone(struct buf *bp)
+uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
 {
-	int npages = bp->b_bufsize >> PAGE_SHIFT;
-	struct vm_page *pg, *pgs[npages];
 	struct uvm_object *uobj;
+	struct vm_page *pg;
 	kmutex_t *slock;
-	int i, error, swslot;
+	int pageout_done;
-	int pageout_done = 0;
+	int swslot;
-	bool write, swap;
+	int i;
-	UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist);
+	bool swap;
-	UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0);
-
-	error = bp->b_error;
-	write = (bp->b_flags & B_READ) == 0;
-	/* XXXUBC BC_NOCACHE is for swap pager, should be done differently */
-	if (write && !(bp->b_cflags & BC_NOCACHE) && bioopsp != NULL)
-		(*bioopsp->io_pageiodone)(bp);
-
-	uobj = NULL;
-	for (i = 0; i < npages; i++) {
-		pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT));
-		UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0);
-	}
-	uvm_pagermapout((vaddr_t)bp->b_data, npages);
 
 	swslot = 0;
+	pageout_done = 0;
 	slock = NULL;
+	uobj = NULL;
 	pg = pgs[0];
 	swap = (pg->uanon != NULL && pg->uobject == NULL) ||
 		(pg->pqflags & PQ_AOBJ) != 0;
@@ -484,6 +466,37 @@ uvm_aio_aiodone(struct buf *bp)
 		uvmexp.pdpending--;
 #endif /* defined(VMSWAP) */
 	}
+}
+
+/*
+ * uvm_aio_aiodone: do iodone processing for async i/os.
+ * this should be called in thread context, not interrupt context.
+ */
+
+void
+uvm_aio_aiodone(struct buf *bp)
+{
+	int npages = bp->b_bufsize >> PAGE_SHIFT;
+	struct vm_page *pgs[npages];
+	int i, error;
+	bool write;
+	UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist);
+	UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0);
+
+	error = bp->b_error;
+	write = (bp->b_flags & B_READ) == 0;
+	/* XXXUBC BC_NOCACHE is for swap pager, should be done differently */
+	if (write && !(bp->b_cflags & BC_NOCACHE) && bioopsp != NULL)
+		(*bioopsp->io_pageiodone)(bp);
+
+	for (i = 0; i < npages; i++) {
+		pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT));
+		UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0);
+	}
+	uvm_pagermapout((vaddr_t)bp->b_data, npages);
+
+	uvm_aio_aiodone_pages(pgs, npages, write, error);
+
 	if (write && (bp->b_cflags & BC_AGE) != 0) {
 		mutex_enter(bp->b_objlock);
 		vwakeup(bp);

sys/uvm/uvm_pdaemon.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_pdaemon.c,v 1.91 2008/02/07 12:24:16 yamt Exp $	*/
+/*	$NetBSD: uvm_pdaemon.c,v 1.92 2008/02/29 20:35:23 yamt Exp $	*/
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -71,7 +71,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.91 2008/02/07 12:24:16 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.92 2008/02/29 20:35:23 yamt Exp $");
 
 #include "opt_uvmhist.h"
 #include "opt_readahead.h"
@@ -537,7 +537,7 @@ swapcluster_flush(struct swapcluster *swc, bool now)
 		uvmexp.pdpageouts++;
 		uvm_pageout_start(nused);
 		error = uvm_swap_put(slot, swc->swc_pages, nused, 0);
-		KASSERT(error == 0);
+		KASSERT(error == 0 || error == ENOMEM);
 	}
 
 	/*

sys/uvm/uvm_swap.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_swap.c,v 1.136 2008/01/30 14:25:21 hannken Exp $	*/
+/*	$NetBSD: uvm_swap.c,v 1.137 2008/02/29 20:35:23 yamt Exp $	*/
 
 /*
  * Copyright (c) 1995, 1996, 1997 Matthew R. Green
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.136 2008/01/30 14:25:21 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.137 2008/02/29 20:35:23 yamt Exp $");
 
 #include "fs_nfs.h"
 #include "opt_uvmhist.h"
@@ -1699,6 +1699,17 @@ uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags)
 	write = (flags & B_READ) == 0;
 	async = (flags & B_ASYNC) != 0;
 
+	/*
+	 * allocate a buf for the i/o.
+	 */
+
+	KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async));
+	bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp);
+	if (bp == NULL) {
+		uvm_aio_aiodone_pages(pps, npages, true, ENOMEM);
+		return ENOMEM;
+	}
+
 	/*
 	 * convert starting drum slot to block number
 	 */
@@ -1714,12 +1725,6 @@ uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags)
 		UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE;
 	kva = uvm_pagermapin(pps, npages, mapinflags);
 
-	/*
-	 * now allocate a buf for the i/o.
-	 */
-
-	bp = getiobuf(swapdev_vp, true);
-
 	/*
 	 * fill in the bp/sbp.   we currently route our i/o through
 	 * /dev/drum's vnode [swapdev_vp].