Modified the dksubr routines to:

o  expect the disk's start routine to return an int.  If the
	   int is non-zero, we enqueue the request and try again
	   later.
	o  have a dk_start() routine which runs the request queue.
	o  have a dk_iodone() function which should be called by the
	   driver using the framwork from its iodone.  dk_iodone will
	   retry the queue since presumably further progress may be
	   possible once a request is complete.  It is required that
	   the underlying driver have the resources to keep at least
	   one transaction in flight at any time.

Modified cgd to:

	o  be able to keep one transaction in flight at any time
	   (almost) by keeping a buffer of size MAXPHYS in its softc
	   and use it.

We still need to make the cgd_cbufpool per device rather than global
and provide a low water mark for it.

Addresses PR: kern/24715
(at least according to the submitter.)

sys/dev/cgd.c

@@ -1,4 +1,4 @@
-/* $NetBSD: cgd.c,v 1.15 2004/03/18 10:42:08 dan Exp $ */
+/* $NetBSD: cgd.c,v 1.16 2004/03/27 23:23:06 elric Exp $ */
 
 /*-
  * Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.15 2004/03/18 10:42:08 dan Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cgd.c,v 1.16 2004/03/27 23:23:06 elric Exp $");
 
 #include <sys/types.h>
 #include <sys/param.h>
@@ -84,7 +84,7 @@ const struct cdevsw cgd_cdevsw = {
 
 /* Internal Functions */
 
-static void	cgdstart(struct dk_softc *, struct buf *);
+static int	cgdstart(struct dk_softc *, struct buf *);
 static void	cgdiodone(struct buf *);
 
 static int	cgd_ioctl_set(struct cgd_softc *, void *, struct proc *);
@@ -182,6 +182,7 @@ cgdsoftc_init(struct cgd_softc *cs, int num)
 
 	memset(cs, 0x0, sizeof(*cs));
 	snprintf(buf, DK_XNAME_SIZE, "cgd%d", num);
+	simple_lock_init(&cs->sc_slock);
 	dk_sc_init(&cs->sc_dksc, cs, buf);
 }
 
@@ -255,7 +256,48 @@ cgdsize(dev_t dev)
 	return dk_size(di, &cs->sc_dksc, dev);
 }
 
+/*
+ * cgd_{get,put}data are functions that deal with getting a buffer
+ * for the new encrypted data.  We have a buffer per device so that
+ * we can ensure that we can always have a transaction in flight.
+ * We use this buffer first so that we have one less piece of
+ * malloc'ed data at any given point.
+ */
+
+static void *
+cgd_getdata(struct dk_softc *dksc, unsigned long size)
+{
+	struct	cgd_softc *cs =dksc->sc_osc;
+	caddr_t	data = NULL;
+
+	simple_lock(&cs->sc_slock);
+	if (cs->sc_data_used == 0) {
+		cs->sc_data_used = 1;
+		data = cs->sc_data;
+	}
+	simple_unlock(&cs->sc_slock);
+
+	if (data)
+		return data;
+
+	return malloc(size, M_DEVBUF, M_NOWAIT);
+}
+
 static void
+cgd_putdata(struct dk_softc *dksc, caddr_t data)
+{
+	struct	cgd_softc *cs =dksc->sc_osc;
+
+	if (data == cs->sc_data) {
+		simple_lock(&cs->sc_slock);
+		cs->sc_data_used = 0;
+		simple_unlock(&cs->sc_slock);
+	} else {
+		free(data, M_DEVBUF);
+	}
+}
+
+static int
 cgdstart(struct dk_softc *dksc, struct buf *bp)
 {
 	struct	cgd_softc *cs = dksc->sc_osc;
@@ -280,32 +322,34 @@ cgdstart(struct dk_softc *dksc, struct buf *bp)
 		bn += pp->p_offset;
 	}
 
+	/*
+	 * We attempt to allocate all of our resources up front, so that
+	 * we can fail quickly if they are unavailable.
+	 */
+
+	cbp = CGD_GETBUF();
+	if (cbp == NULL) {
+		disk_unbusy(&dksc->sc_dkdev, 0, (bp->b_flags & B_READ));
+		return -1;
+	}
+
 	/*
 	 * If we are writing, then we need to encrypt the outgoing
-	 * block.  In the best case scenario, we are able to allocate
+	 * block into a new block of memory.  If we fail, then we
-	 * enough memory to encrypt the data in a new block, otherwise
+	 * return an error and let the dksubr framework deal with it.
-	 * we encrypt it in place (noting we'll have to decrypt it after
-	 * the write.)
 	 */
 	newaddr = addr = bp->b_data;
 	if ((bp->b_flags & B_READ) == 0) {
-		newaddr = malloc(bp->b_bcount, M_DEVBUF, 0);
+		newaddr = cgd_getdata(dksc, bp->b_bcount);
-		if (!newaddr)
+		if (!newaddr) {
-			newaddr = addr;
+			CGD_PUTBUF(cbp);
+			disk_unbusy(&dksc->sc_dkdev, 0, (bp->b_flags & B_READ));
+			return -1;
+		}
 		cgd_cipher(cs, newaddr, addr, bp->b_bcount, bn,
 		    DEV_BSIZE, CGD_CIPHER_ENCRYPT);
 	}
 
-	cbp = CGD_GETBUF();
-	if (cbp == NULL) {
-		bp->b_error = ENOMEM;
-		bp->b_flags |= B_ERROR;
-		if (newaddr != addr)
-			free(newaddr, M_DEVBUF);
-		biodone(bp);
-		disk_unbusy(&dksc->sc_dkdev, 0, (bp->b_flags & B_READ));
-		return;
-	}
 	BUF_INIT(&cbp->cb_buf);
 	cbp->cb_buf.b_data = newaddr;
 	cbp->cb_buf.b_flags = bp->b_flags | B_CALL;
@@ -324,6 +368,7 @@ cgdstart(struct dk_softc *dksc, struct buf *bp)
 	if ((cbp->cb_buf.b_flags & B_READ) == 0)
 		cbp->cb_buf.b_vp->v_numoutput++;
 	VOP_STRATEGY(cs->sc_tvn, &cbp->cb_buf);
+	return 0;
 }
 
 void
@@ -350,21 +395,19 @@ cgdiodone(struct buf *vbp)
 		printf("%s: error %d\n", dksc->sc_xname, obp->b_error);
 	}
 
-	/* Perform the decryption if we need to:
+	/* Perform the decryption if we are reading.
-	 *	o  if we are reading, or
-	 *	o  we wrote and couldn't allocate memory.
 	 *
 	 * Note: use the blocknumber from nbp, since it is what
 	 *       we used to encrypt the blocks.
 	 */
 
-	if (nbp->b_flags & B_READ || nbp->b_data == obp->b_data)
+	if (nbp->b_flags & B_READ)
 		cgd_cipher(cs, obp->b_data, obp->b_data, obp->b_bcount,
 		    nbp->b_blkno, DEV_BSIZE, CGD_CIPHER_DECRYPT);
 
-	/* If we managed to allocate memory, free it now... */
+	/* If we allocated memory, free it now... */
 	if (nbp->b_data != obp->b_data)
-		free(nbp->b_data, M_DEVBUF);
+		cgd_putdata(dksc, nbp->b_data);
 
 	CGD_PUTBUF(cbp);
 
@@ -375,6 +418,7 @@ cgdiodone(struct buf *vbp)
 	disk_unbusy(&dksc->sc_dkdev, obp->b_bcount - obp->b_resid,
 	    (obp->b_flags & B_READ));
 	biodone(obp);
+	dk_iodone(di, dksc);
 	splx(s);
 }
 
@@ -536,6 +580,11 @@ cgd_ioctl_set(struct cgd_softc *cs, void *data, struct proc *p)
 		goto bail;
 	}
 
+	bufq_alloc(&cs->sc_dksc.sc_bufq, BUFQ_FCFS);
+
+	cs->sc_data = malloc(MAXPHYS, M_DEVBUF, M_WAITOK);
+	cs->sc_data_used = 0;
+
 	cs->sc_dksc.sc_flags |= DKF_INITED;
 
 	/* Attach the disk. */
@@ -555,10 +604,25 @@ bail:
 static int
 cgd_ioctl_clr(struct cgd_softc *cs, void *data, struct proc *p)
 {
+	struct	buf *bp;
+	int	s;
+
+	/* Kill off any queued buffers. */
+	s = splbio();
+	while ((bp = BUFQ_GET(&cs->sc_dksc.sc_bufq)) != NULL) {
+		bp->b_error = EIO;
+		bp->b_flags |= B_ERROR;
+		bp->b_resid = bp->b_bcount;
+		biodone(bp);
+	}
+	splx(s);
+	bufq_free(&cs->sc_dksc.sc_bufq);
 
 	(void)vn_close(cs->sc_tvn, FREAD|FWRITE, p->p_ucred, p);
 	cs->sc_cfuncs->cf_destroy(cs->sc_cdata.cf_priv);
 	free(cs->sc_tpath, M_DEVBUF);
+	free(cs->sc_data, M_DEVBUF);
+	cs->sc_data_used = 0;
 	cs->sc_dksc.sc_flags &= ~DKF_INITED;
 	disk_detach(&cs->sc_dksc.sc_dkdev);
 

sys/dev/cgdvar.h

@@ -1,4 +1,4 @@
-/* $NetBSD: cgdvar.h,v 1.1 2002/10/04 18:22:35 elric Exp $ */
+/* $NetBSD: cgdvar.h,v 1.2 2004/03/27 23:23:06 elric Exp $ */
 
 /*-
  * Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -71,9 +71,12 @@ struct cgd_softc {
 	struct vnode		*sc_tvn;	/* target device's vnode */
 	dev_t			 sc_tdev;	/* target device */
 	char			*sc_tpath;	/* target device's path */
+	caddr_t			 sc_data;	/* emergency buffer */
+	int			 sc_data_used;	/* Really lame, we'll change */
 	size_t			 sc_tpathlen;	/* length of prior string */
 	struct cryptdata	 sc_cdata;	/* crypto data */
 	struct cryptfuncs	*sc_cfuncs;	/* encryption functions */
+	struct simplelock	 sc_slock;	/* our lock */
 };
 #endif
 

sys/dev/dksubr.c

@@ -1,4 +1,4 @@
-/* $NetBSD: dksubr.c,v 1.10 2003/07/14 15:47:03 lukem Exp $ */
+/* $NetBSD: dksubr.c,v 1.11 2004/03/27 23:23:06 elric Exp $ */
 
 /*-
  * Copyright (c) 1996, 1997, 1998, 1999, 2002 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: dksubr.c,v 1.10 2003/07/14 15:47:03 lukem Exp $");
+__KERNEL_RCSID(0, "$NetBSD: dksubr.c,v 1.11 2004/03/27 23:23:06 elric Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -207,11 +207,38 @@ dk_strategy(struct dk_intf *di, struct dk_softc *dksc, struct buf *bp)
 	 * provided by the individual driver.
 	 */
 	s = splbio();
-	di->di_diskstart(dksc, bp);
+	BUFQ_PUT(&dksc->sc_bufq, bp);
+	dk_start(di, dksc);
 	splx(s);
 	return;
 }
 
+void
+dk_start(struct dk_intf *di, struct dk_softc *dksc)
+{
+	struct	buf *bp;
+
+	DPRINTF_FOLLOW(("dk_start(%s, %p)\n", di->di_dkname, dksc));
+
+	/* Process the work queue */
+	while ((bp = BUFQ_PEEK(&dksc->sc_bufq)) != NULL) {
+		if (di->di_diskstart(dksc, bp) != -1)
+			(void) BUFQ_GET(&dksc->sc_bufq);
+		else
+			break;
+	}
+}
+
+void
+dk_iodone(struct dk_intf *di, struct dk_softc *dksc)
+{
+
+	DPRINTF_FOLLOW(("dk_iodone(%s, %p)\n", di->di_dkname, dksc));
+
+	/* We kick the queue in case we are able to get more work done */
+	dk_start(di, dksc);
+}
+
 int
 dk_size(struct dk_intf *di, struct dk_softc *dksc, dev_t dev)
 {

sys/dev/dkvar.h

@@ -1,4 +1,4 @@
-/* $NetBSD: dkvar.h,v 1.3 2003/06/29 22:29:59 fvdl Exp $ */
+/* $NetBSD: dkvar.h,v 1.4 2004/03/27 23:23:06 elric Exp $ */
 
 /*-
  * Copyright (c) 2002 The NetBSD Foundation, Inc.
@@ -55,11 +55,12 @@ struct dk_softc {
 						 * driver */
 	u_int32_t		 sc_flags;	/* flags */
 	size_t			 sc_size;	/* size of disk */
-	struct dk_geom	 sc_geom;		/* geometry info */
+	struct dk_geom		 sc_geom;	/* geometry info */
 #define DK_XNAME_SIZE 8
 	char			 sc_xname[DK_XNAME_SIZE]; /* external name */
 	struct disk		 sc_dkdev;	/* generic disk info */
 	struct lock		 sc_lock;	/* the lock */
+	struct bufq_state	 sc_bufq;	/* buffer queue */
 };
 
 /* sc_flags:
@@ -88,7 +89,7 @@ struct dk_intf {
 	int	(*di_open)(dev_t, int, int, struct proc *);
 	int	(*di_close)(dev_t, int, int, struct proc *);
 	void	(*di_strategy)(struct buf *);
-	void	(*di_diskstart)(struct dk_softc *, struct buf *);
+	int	(*di_diskstart)(struct dk_softc *, struct buf *);
 };
 
 #define DK_BUSY(_dksc, _pmask)				\
@@ -107,6 +108,8 @@ int	dk_open(struct dk_intf *, struct dk_softc *, dev_t,
 int	dk_close(struct dk_intf *, struct dk_softc *, dev_t,
 		 int, int, struct proc *);
 void	dk_strategy(struct dk_intf *, struct dk_softc *, struct buf *);
+void	dk_start(struct dk_intf *, struct dk_softc *);
+void	dk_iodone(struct dk_intf *, struct dk_softc *);
 int	dk_size(struct dk_intf *, struct dk_softc *, dev_t);
 int	dk_ioctl(struct dk_intf *, struct dk_softc *, dev_t,
 		 u_long, caddr_t, int, struct proc *);