From 7266a9590738a6bf96c473667d1bd39b93c47617 Mon Sep 17 00:00:00 2001 From: yamt Date: Sat, 10 Jan 2004 14:39:50 +0000 Subject: [PATCH] store a i/o priority hint in struct buf for buffer queue discipline. --- sys/dev/ata/wd.c | 7 +++++-- sys/dev/ccd.c | 6 ++++-- sys/dev/cgd.c | 6 ++++-- sys/dev/raidframe/rf_netbsdkintf.c | 6 ++++-- sys/dev/scsipi/cd.c | 7 +++++-- sys/dev/vnd.c | 6 ++++-- sys/kern/kern_physio.c | 6 ++++-- sys/kern/vfs_bio.c | 15 +++++++++++++-- sys/miscfs/genfs/genfs_vnops.c | 14 ++++++++++++-- sys/sys/buf.h | 14 +++++++++++++- sys/ufs/ffs/ffs_inode.c | 5 +++-- sys/ufs/lfs/lfs_segment.c | 12 ++++++++++-- sys/ufs/ufs/ufs_bmap.c | 5 +++-- sys/uvm/uvm_swap.c | 10 ++++++++-- 14 files changed, 92 insertions(+), 27 deletions(-) diff --git a/sys/dev/ata/wd.c b/sys/dev/ata/wd.c index a5ffeabbf73e..9f6535a66a72 100644 --- a/sys/dev/ata/wd.c +++ b/sys/dev/ata/wd.c @@ -1,4 +1,4 @@ -/* $NetBSD: wd.c,v 1.272 2003/12/14 05:38:20 thorpej Exp $ */ +/* $NetBSD: wd.c,v 1.273 2004/01/10 14:39:50 yamt Exp $ */ /* * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. @@ -66,7 +66,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: wd.c,v 1.272 2003/12/14 05:38:20 thorpej Exp $"); +__KERNEL_RCSID(0, "$NetBSD: wd.c,v 1.273 2004/01/10 14:39:50 yamt Exp $"); #ifndef WDCDEBUG #define WDCDEBUG @@ -684,6 +684,9 @@ __wdstart(struct wd_softc *wd, struct buf *bp) /* Put ptr to orig buf in b_private and use new buf */ nbp->b_private = bp; + + BIO_COPYPRIO(nbp, bp); + bp = nbp; } diff --git a/sys/dev/ccd.c b/sys/dev/ccd.c index 9c4b62f6851a..2d63130abe65 100644 --- a/sys/dev/ccd.c +++ b/sys/dev/ccd.c @@ -1,4 +1,4 @@ -/* $NetBSD: ccd.c,v 1.93 2003/10/26 23:42:57 chs Exp $ */ +/* $NetBSD: ccd.c,v 1.94 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1996, 1997, 1998, 1999 The NetBSD Foundation, Inc. @@ -125,7 +125,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: ccd.c,v 1.93 2003/10/26 23:42:57 chs Exp $"); +__KERNEL_RCSID(0, "$NetBSD: ccd.c,v 1.94 2004/01/10 14:39:50 yamt Exp $"); #include #include @@ -870,6 +870,8 @@ ccdbuffer(cs, bp, bn, addr, bcount) cbp->cb_sc = cs; cbp->cb_comp = ccdisk; + BIO_COPYPRIO(&cbp->cb_buf, bp); + #ifdef DEBUG if (ccddebug & CCDB_IO) printf(" dev 0x%x(u%lu): cbp %p bn %" PRId64 " addr %p" diff --git a/sys/dev/cgd.c b/sys/dev/cgd.c index 2517222c4ea4..7cefc894839c 100644 --- a/sys/dev/cgd.c +++ b/sys/dev/cgd.c @@ -1,4 +1,4 @@ -/* $NetBSD: cgd.c,v 1.12 2003/06/29 22:29:58 fvdl Exp $ */ +/* $NetBSD: cgd.c,v 1.13 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 2002 The NetBSD Foundation, Inc. @@ -37,7 +37,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.12 2003/06/29 22:29:58 fvdl Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.13 2004/01/10 14:39:50 yamt Exp $"); #include #include @@ -320,6 +320,8 @@ cgdstart(struct dk_softc *dksc, struct buf *bp) cbp->cb_obp = bp; cbp->cb_sc = cs; + BIO_COPYPRIO(&cbp->cb_buf, bp); + if ((cbp->cb_buf.b_flags & B_READ) == 0) cbp->cb_buf.b_vp->v_numoutput++; VOP_STRATEGY(&cbp->cb_buf); diff --git a/sys/dev/raidframe/rf_netbsdkintf.c b/sys/dev/raidframe/rf_netbsdkintf.c index 16343ec3bb31..5f3b30f81b74 100644 --- a/sys/dev/raidframe/rf_netbsdkintf.c +++ b/sys/dev/raidframe/rf_netbsdkintf.c @@ -1,4 +1,4 @@ -/* $NetBSD: rf_netbsdkintf.c,v 1.171 2004/01/04 21:06:04 oster Exp $ */ +/* $NetBSD: rf_netbsdkintf.c,v 1.172 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. * All rights reserved. @@ -146,7 +146,7 @@ ***********************************************************/ #include -__KERNEL_RCSID(0, "$NetBSD: rf_netbsdkintf.c,v 1.171 2004/01/04 21:06:04 oster Exp $"); +__KERNEL_RCSID(0, "$NetBSD: rf_netbsdkintf.c,v 1.172 2004/01/10 14:39:50 yamt Exp $"); #include #include @@ -1830,6 +1830,8 @@ rf_DispatchKernelIO(RF_DiskQueue_t *queue, RF_DiskQueueData_t *req) raidbp->rf_obp = bp; raidbp->req = req; + BIO_COPYPRIO(&raidbp->rf_buf, bp); + switch (req->type) { case RF_IO_TYPE_NOP: /* used primarily to unlock a locked queue */ /* XXX need to do something extra here.. */ diff --git a/sys/dev/scsipi/cd.c b/sys/dev/scsipi/cd.c index 0d7c58e4d66e..b1eaa4fc63db 100644 --- a/sys/dev/scsipi/cd.c +++ b/sys/dev/scsipi/cd.c @@ -1,4 +1,4 @@ -/* $NetBSD: cd.c,v 1.198 2003/11/10 08:51:52 wiz Exp $ */ +/* $NetBSD: cd.c,v 1.199 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1998, 2001, 2003 The NetBSD Foundation, Inc. @@ -54,7 +54,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: cd.c,v 1.198 2003/11/10 08:51:52 wiz Exp $"); +__KERNEL_RCSID(0, "$NetBSD: cd.c,v 1.199 2004/01/10 14:39:50 yamt Exp $"); #include "rnd.h" @@ -722,6 +722,9 @@ cdstrategy(bp) /* Put ptr to orig buf in b_private and use new buf */ nbp->b_private = bp; + + BIO_COPYPRIO(nbp, bp); + bp = nbp; } else { diff --git a/sys/dev/vnd.c b/sys/dev/vnd.c index b7fe63f85f26..6ec23207a385 100644 --- a/sys/dev/vnd.c +++ b/sys/dev/vnd.c @@ -1,4 +1,4 @@ -/* $NetBSD: vnd.c,v 1.104 2003/10/19 07:41:19 scw Exp $ */ +/* $NetBSD: vnd.c,v 1.105 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. @@ -133,7 +133,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: vnd.c,v 1.104 2003/10/19 07:41:19 scw Exp $"); +__KERNEL_RCSID(0, "$NetBSD: vnd.c,v 1.105 2004/01/10 14:39:50 yamt Exp $"); #if defined(_KERNEL_OPT) #include "fs_nfs.h" @@ -563,6 +563,8 @@ vndstrategy(bp) nbp->vb_xfer = vnx; + BIO_COPYPRIO(&nbp->vb_buf, bp); + /* * Just sort by block number */ diff --git a/sys/kern/kern_physio.c b/sys/kern/kern_physio.c index 518a7d3d9562..189b7b6e0e4a 100644 --- a/sys/kern/kern_physio.c +++ b/sys/kern/kern_physio.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_physio.c,v 1.57 2003/08/07 16:31:47 agc Exp $ */ +/* $NetBSD: kern_physio.c,v 1.58 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1982, 1986, 1990, 1993 @@ -71,7 +71,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.57 2003/08/07 16:31:47 agc Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.58 2004/01/10 14:39:50 yamt Exp $"); #include #include @@ -208,6 +208,8 @@ physio(strategy, bp, dev, flags, minphys, uio) } vmapbuf(bp, todo); + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); + /* [call strategy to start the transfer] */ (*strategy)(bp); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 6d605baeae19..8c5389a9a5dd 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -1,4 +1,4 @@ -/* $NetBSD: vfs_bio.c,v 1.107 2004/01/09 19:01:01 thorpej Exp $ */ +/* $NetBSD: vfs_bio.c,v 1.108 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1982, 1986, 1989, 1993 @@ -81,7 +81,7 @@ #include "opt_softdep.h" #include -__KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.107 2004/01/09 19:01:01 thorpej Exp $"); +__KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.108 2004/01/10 14:39:50 yamt Exp $"); #include #include @@ -502,6 +502,10 @@ bio_doread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { /* Start I/O for the buffer. */ SET(bp->b_flags, B_READ | async); + if (async) + BIO_SETPRIO(bp, BPRIO_TIMELIMITED); + else + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); VOP_STRATEGY(bp); /* Pay for the read. */ @@ -642,6 +646,11 @@ bwrite(struct buf *bp) simple_unlock(&bp->b_interlock); splx(s); + if (sync) + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); + else + BIO_SETPRIO(bp, BPRIO_TIMELIMITED); + VOP_STRATEGY(bp); if (sync) { @@ -978,6 +987,7 @@ start: } else { allocbuf(bp, size, preserve); } + BIO_SETPRIO(bp, BPRIO_DEFAULT); return (bp); } @@ -1273,6 +1283,7 @@ biodone(struct buf *bp) if (ISSET(bp->b_flags, B_DONE)) panic("biodone already"); SET(bp->b_flags, B_DONE); /* note that it's done */ + BIO_SETPRIO(bp, BPRIO_DEFAULT); if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete) (*bioops.io_complete)(bp); diff --git a/sys/miscfs/genfs/genfs_vnops.c b/sys/miscfs/genfs/genfs_vnops.c index 7aadcab1949c..71ffe96abe61 100644 --- a/sys/miscfs/genfs/genfs_vnops.c +++ b/sys/miscfs/genfs/genfs_vnops.c @@ -1,4 +1,4 @@ -/* $NetBSD: genfs_vnops.c,v 1.83 2003/11/27 07:58:02 pk Exp $ */ +/* $NetBSD: genfs_vnops.c,v 1.84 2004/01/10 14:39:50 yamt Exp $ */ /* * Copyright (c) 1982, 1986, 1989, 1993 @@ -31,7 +31,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: genfs_vnops.c,v 1.83 2003/11/27 07:58:02 pk Exp $"); +__KERNEL_RCSID(0, "$NetBSD: genfs_vnops.c,v 1.84 2004/01/10 14:39:50 yamt Exp $"); #include "opt_nfsserver.h" @@ -822,6 +822,10 @@ genfs_getpages(void *v) "bp %p offset 0x%x bcount 0x%x blkno 0x%x", bp, offset, iobytes, bp->b_blkno); + if (async) + BIO_SETPRIO(bp, BPRIO_TIMELIMITED); + else + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); VOP_STRATEGY(bp); } @@ -1489,6 +1493,12 @@ genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) UVMHIST_LOG(ubchist, "vp %p offset 0x%x bcount 0x%x blkno 0x%x", vp, offset, bp->b_bcount, bp->b_blkno); + if (curproc == uvm.pagedaemon_proc) + BIO_SETPRIO(bp, BPRIO_TIMELIMITED); + else if (async) + BIO_SETPRIO(bp, BPRIO_TIMENONCRITICAL); + else + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); VOP_STRATEGY(bp); } if (skipbytes) { diff --git a/sys/sys/buf.h b/sys/sys/buf.h index 413c28561431..19727b3ca576 100644 --- a/sys/sys/buf.h +++ b/sys/sys/buf.h @@ -1,4 +1,4 @@ -/* $NetBSD: buf.h,v 1.68 2003/12/30 20:44:16 thorpej Exp $ */ +/* $NetBSD: buf.h,v 1.69 2004/01/10 14:39:50 yamt Exp $ */ /*- * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. @@ -153,6 +153,7 @@ struct buf { struct simplelock b_interlock; /* Lock for b_flags changes */ volatile long b_flags; /* B_* flags. */ int b_error; /* Errno value. */ + int b_prio; /* Hint for buffer queue discipline. */ long b_bufsize; /* Allocated buffer size. */ long b_bcount; /* Valid bytes in buffer. */ long b_resid; /* Remaining I/O. */ @@ -197,6 +198,7 @@ struct buf { do { \ LIST_INIT(&(bp)->b_dep); \ simple_lock_init(&(bp)->b_interlock); \ + BIO_SETPRIO((bp), BPRIO_DEFAULT); \ } while (/*CONSTCOND*/0) /* @@ -265,6 +267,16 @@ do { \ #ifdef _KERNEL +#define BIO_GETPRIO(bp) ((bp)->b_prio) +#define BIO_SETPRIO(bp, prio) (bp)->b_prio = (prio) +#define BIO_COPYPRIO(bp1, bp2) BIO_SETPRIO(bp1, BIO_GETPRIO(bp2)) + +#define BPRIO_NPRIO 3 +#define BPRIO_TIMECRITICAL 2 +#define BPRIO_TIMELIMITED 1 +#define BPRIO_TIMENONCRITICAL 0 +#define BPRIO_DEFAULT BPRIO_TIMELIMITED + extern struct bio_ops bioops; extern u_int nbuf; /* The number of buffer headers */ extern struct buf *buf; /* The buffer headers. */ diff --git a/sys/ufs/ffs/ffs_inode.c b/sys/ufs/ffs/ffs_inode.c index 2a7a289e715e..1407669226cc 100644 --- a/sys/ufs/ffs/ffs_inode.c +++ b/sys/ufs/ffs/ffs_inode.c @@ -1,4 +1,4 @@ -/* $NetBSD: ffs_inode.c,v 1.60 2003/08/07 16:34:30 agc Exp $ */ +/* $NetBSD: ffs_inode.c,v 1.61 2004/01/10 14:39:51 yamt Exp $ */ /* * Copyright (c) 1982, 1986, 1989, 1993 @@ -32,7 +32,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: ffs_inode.c,v 1.60 2003/08/07 16:34:30 agc Exp $"); +__KERNEL_RCSID(0, "$NetBSD: ffs_inode.c,v 1.61 2004/01/10 14:39:51 yamt Exp $"); #if defined(_KERNEL_OPT) #include "opt_ffs.h" @@ -563,6 +563,7 @@ ffs_indirtrunc(ip, lbn, dbn, lastbn, level, countp) if (bp->b_bcount > bp->b_bufsize) panic("ffs_indirtrunc: bad buffer size"); bp->b_blkno = dbn; + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); VOP_STRATEGY(bp); error = biowait(bp); } diff --git a/sys/ufs/lfs/lfs_segment.c b/sys/ufs/lfs/lfs_segment.c index 297066a7c18d..1e632bd36e51 100644 --- a/sys/ufs/lfs/lfs_segment.c +++ b/sys/ufs/lfs/lfs_segment.c @@ -1,4 +1,4 @@ -/* $NetBSD: lfs_segment.c,v 1.146 2003/12/17 10:38:39 yamt Exp $ */ +/* $NetBSD: lfs_segment.c,v 1.147 2004/01/10 14:39:51 yamt Exp $ */ /*- * Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc. @@ -67,7 +67,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.146 2003/12/17 10:38:39 yamt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.147 2004/01/10 14:39:51 yamt Exp $"); #define ivndebug(vp,str) printf("ino %d: %s\n",VTOI(vp)->i_number,(str)) @@ -2000,6 +2000,10 @@ lfs_writeseg(struct lfs *fs, struct segment *sp) bpp++; i--; } + if (fs->lfs_sp->seg_flags & SEGM_SYNC) + BIO_SETPRIO(cbp, BPRIO_TIMECRITICAL); + else + BIO_SETPRIO(cbp, BPRIO_TIMELIMITED); s = splbio(); V_INCR_NUMOUTPUT(devvp); splx(s); @@ -2063,6 +2067,10 @@ lfs_writesuper(struct lfs *fs, daddr_t daddr) bp->b_flags &= ~(B_DONE | B_ERROR | B_READ | B_DELWRI); bp->b_iodone = lfs_supercallback; + if (fs->lfs_sp != NULL && fs->lfs_sp->seg_flags & SEGM_SYNC) + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); + else + BIO_SETPRIO(bp, BPRIO_TIMELIMITED); vop_strategy_a.a_desc = VDESC(vop_strategy); vop_strategy_a.a_bp = bp; curproc->p_stats->p_ru.ru_oublock++; diff --git a/sys/ufs/ufs/ufs_bmap.c b/sys/ufs/ufs/ufs_bmap.c index 19084a4b5c19..d56cbc641d17 100644 --- a/sys/ufs/ufs/ufs_bmap.c +++ b/sys/ufs/ufs/ufs_bmap.c @@ -1,4 +1,4 @@ -/* $NetBSD: ufs_bmap.c,v 1.25 2003/08/07 16:34:44 agc Exp $ */ +/* $NetBSD: ufs_bmap.c,v 1.26 2004/01/10 14:39:51 yamt Exp $ */ /* * Copyright (c) 1989, 1991, 1993 @@ -37,7 +37,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: ufs_bmap.c,v 1.25 2003/08/07 16:34:44 agc Exp $"); +__KERNEL_RCSID(0, "$NetBSD: ufs_bmap.c,v 1.26 2004/01/10 14:39:51 yamt Exp $"); #include #include @@ -243,6 +243,7 @@ ufs_bmaparray(vp, bn, bnp, ap, nump, runp, is_sequential) trace(TR_BREADMISS, pack(vp, size), metalbn); bp->b_blkno = blkptrtodb(ump, daddr); bp->b_flags |= B_READ; + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); VOP_STRATEGY(bp); curproc->p_stats->p_ru.ru_inblock++; /* XXX */ if ((error = biowait(bp)) != 0) { diff --git a/sys/uvm/uvm_swap.c b/sys/uvm/uvm_swap.c index 62d0584f5589..544a68aabaab 100644 --- a/sys/uvm/uvm_swap.c +++ b/sys/uvm/uvm_swap.c @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_swap.c,v 1.82 2003/08/28 13:12:20 pk Exp $ */ +/* $NetBSD: uvm_swap.c,v 1.83 2004/01/10 14:39:51 yamt Exp $ */ /* * Copyright (c) 1995, 1996, 1997 Matthew R. Green @@ -32,7 +32,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.82 2003/08/28 13:12:20 pk Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.83 2004/01/10 14:39:51 yamt Exp $"); #include "fs_nfs.h" #include "opt_uvmhist.h" @@ -1749,6 +1749,12 @@ uvm_swap_io(pps, startslot, npages, flags) bp->b_flags |= B_CALL; bp->b_iodone = uvm_aio_biodone; UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); + if (curproc == uvm.pagedaemon_proc) + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); + else + BIO_SETPRIO(bp, BPRIO_TIMELIMITED); + } else { + BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); } UVMHIST_LOG(pdhist, "about to start io: data = %p blkno = 0x%x, bcount = %ld",