From 0585ce14890b4b3a02d1275575d548a1628eb049 Mon Sep 17 00:00:00 2001 From: thorpej Date: Wed, 3 Jul 2002 19:06:47 +0000 Subject: [PATCH] Make insertion of data into socket buffers O(C): * Keep pointers to the first and last mbufs of the last record in the socket buffer. * Use the sb_lastrecord pointer in the sbappend*() family of functions to avoid traversing the packet chain to find the last record. * Add a new sbappend_stream() function for stream protocols which guarantee that there will never be more than one record in the socket buffer. This function uses the sb_mbtail pointer to perform the data insertion. Make TCP use sbappend_stream(). On a profiling run, this makes sbappend of a TCP transmission using a 1M socket buffer go from 50% of the time to .02% of the time. Thanks to Bill Sommerfeld and YAMAMOTO Takashi for their debugging assistance! --- sys/dev/kttcp.c | 78 +++++++++++++++- sys/kern/uipc_socket.c | 76 ++++++++++++++- sys/kern/uipc_socket2.c | 190 +++++++++++++++++++++++++++++++------- sys/netccitt/if_x25subr.c | 12 ++- sys/netccitt/pk_output.c | 5 +- sys/netccitt/pk_usrreq.c | 5 +- sys/netinet/tcp_input.c | 10 +- sys/netinet/tcp_usrreq.c | 8 +- sys/sys/socketvar.h | 25 ++++- 9 files changed, 349 insertions(+), 60 deletions(-) diff --git a/sys/dev/kttcp.c b/sys/dev/kttcp.c index d1a1e14e4202..059164f8ccf2 100644 --- a/sys/dev/kttcp.c +++ b/sys/dev/kttcp.c @@ -1,4 +1,4 @@ -/* $NetBSD: kttcp.c,v 1.1 2002/06/28 23:27:14 thorpej Exp $ */ +/* $NetBSD: kttcp.c,v 1.2 2002/07/03 19:06:47 thorpej Exp $ */ /* * Copyright (c) 2002 Wasabi Systems, Inc. @@ -268,6 +268,10 @@ kttcp_sosend(struct socket *so, unsigned long long slen, if (space < resid && (atomic || space < so->so_snd.sb_lowat)) { if (so->so_state & SS_NBIO) snderr(EWOULDBLOCK); + SBLASTRECORDCHK(&so->so_rcv, + "kttcp_soreceive sbwait 1"); + SBLASTMBUFCHK(&so->so_rcv, + "kttcp_soreceive sbwait 1"); sbunlock(&so->so_snd); error = sbwait(&so->so_snd); splx(s); @@ -470,10 +474,18 @@ kttcp_soreceive(struct socket *so, unsigned long long slen, goto restart; } dontblock: + /* + * On entry here, m points to the first record of the socket buffer. + * While we process the initial mbufs containing address and control + * info, we save a copy of m->m_nextpkt into nextrecord. + */ #ifdef notyet /* XXXX */ if (uio->uio_procp) uio->uio_procp->p_stats->p_ru.ru_msgrcv++; #endif + KASSERT(m == so->so_rcv.sb_mb); + SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 1"); + SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 1"); nextrecord = m->m_nextpkt; if (pr->pr_flags & PR_ADDR) { #ifdef DIAGNOSTIC @@ -498,13 +510,39 @@ kttcp_soreceive(struct socket *so, unsigned long long slen, m = so->so_rcv.sb_mb; } } + + /* + * If m is non-NULL, we have some data to read. From now on, + * make sure to keep sb_lastrecord consistent when working on + * the last packet on the chain (nextrecord == NULL) and we + * change m->m_nextpkt. + */ if (m) { - if ((flags & MSG_PEEK) == 0) + if ((flags & MSG_PEEK) == 0) { m->m_nextpkt = nextrecord; + /* + * If nextrecord == NULL (this is a single chain), + * then sb_lastrecord may not be valid here if m + * was changed earlier. + */ + if (nextrecord == NULL) { + KASSERT(so->so_rcv.sb_mb == m); + so->so_rcv.sb_lastrecord = m; + } + } type = m->m_type; if (type == MT_OOBDATA) flags |= MSG_OOB; + } else { + if ((flags & MSG_PEEK) == 0) { + KASSERT(so->so_rcv.sb_mb == m); + so->so_rcv.sb_mb = nextrecord; + SB_UPDATE_TAIL(&so->so_rcv); + } } + SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 2"); + SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 2"); + moff = 0; offset = 0; while (m && resid > 0 && error == 0) { @@ -550,8 +588,23 @@ kttcp_soreceive(struct socket *so, unsigned long long slen, MFREE(m, so->so_rcv.sb_mb); m = so->so_rcv.sb_mb; } - if (m) + /* + * If m != NULL, we also know that + * so->so_rcv.sb_mb != NULL. + */ + KASSERT(so->so_rcv.sb_mb == m); + if (m) { m->m_nextpkt = nextrecord; + if (nextrecord == NULL) + so->so_rcv.sb_lastrecord = m; + } else { + so->so_rcv.sb_mb = nextrecord; + SB_UPDATE_TAIL(&so->so_rcv); + } + SBLASTRECORDCHK(&so->so_rcv, + "kttcp_soreceive 3"); + SBLASTMBUFCHK(&so->so_rcv, + "kttcp_soreceive 3"); } } else { if (flags & MSG_PEEK) @@ -590,6 +643,10 @@ kttcp_soreceive(struct socket *so, unsigned long long slen, !sosendallatonce(so) && !nextrecord) { if (so->so_error || so->so_state & SS_CANTRCVMORE) break; + SBLASTRECORDCHK(&so->so_rcv, + "kttcp_soreceive sbwait 2"); + SBLASTMBUFCHK(&so->so_rcv, + "kttcp_soreceive sbwait 2"); error = sbwait(&so->so_rcv); if (error) { sbunlock(&so->so_rcv); @@ -607,8 +664,21 @@ kttcp_soreceive(struct socket *so, unsigned long long slen, (void) sbdroprecord(&so->so_rcv); } if ((flags & MSG_PEEK) == 0) { - if (m == 0) + if (m == 0) { + /* + * First part is an SB_UPDATE_TAIL(). Second part + * makes sure sb_lastrecord is up-to-date if + * there is still data in the socket buffer. + */ so->so_rcv.sb_mb = nextrecord; + if (so->so_rcv.sb_mb == NULL) { + so->so_rcv.sb_mbtail = NULL; + so->so_rcv.sb_lastrecord = NULL; + } else if (nextrecord->m_nextpkt == NULL) + so->so_rcv.sb_lastrecord = nextrecord; + } + SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 4"); + SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 4"); if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) (*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0, (struct mbuf *)(long)flags, (struct mbuf *)0, diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 47e53549214a..a63d4b9438b2 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -1,4 +1,4 @@ -/* $NetBSD: uipc_socket.c,v 1.68 2002/06/11 00:21:33 matt Exp $ */ +/* $NetBSD: uipc_socket.c,v 1.69 2002/07/03 19:06:48 thorpej Exp $ */ /*- * Copyright (c) 2002 The NetBSD Foundation, Inc. @@ -72,7 +72,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.68 2002/06/11 00:21:33 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uipc_socket.c,v 1.69 2002/07/03 19:06:48 thorpej Exp $"); #include "opt_sock_counters.h" #include "opt_sosend_loan.h" @@ -894,6 +894,8 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, error = EWOULDBLOCK; goto release; } + SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1"); sbunlock(&so->so_rcv); error = sbwait(&so->so_rcv); splx(s); @@ -902,10 +904,18 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, goto restart; } dontblock: + /* + * On entry here, m points to the first record of the socket buffer. + * While we process the initial mbufs containing address and control + * info, we save a copy of m->m_nextpkt into nextrecord. + */ #ifdef notyet /* XXXX */ if (uio->uio_procp) uio->uio_procp->p_stats->p_ru.ru_msgrcv++; #endif + KASSERT(m == so->so_rcv.sb_mb); + SBLASTRECORDCHK(&so->so_rcv, "soreceive 1"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 1"); nextrecord = m->m_nextpkt; if (pr->pr_flags & PR_ADDR) { #ifdef DIAGNOSTIC @@ -958,13 +968,39 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, controlp = &(*controlp)->m_next; } } + + /* + * If m is non-NULL, we have some data to read. From now on, + * make sure to keep sb_lastrecord consistent when working on + * the last packet on the chain (nextrecord == NULL) and we + * change m->m_nextpkt. + */ if (m) { - if ((flags & MSG_PEEK) == 0) + if ((flags & MSG_PEEK) == 0) { m->m_nextpkt = nextrecord; + /* + * If nextrecord == NULL (this is a single chain), + * then sb_lastrecord may not be valid here if m + * was changed earlier. + */ + if (nextrecord == NULL) { + KASSERT(so->so_rcv.sb_mb == m); + so->so_rcv.sb_lastrecord = m; + } + } type = m->m_type; if (type == MT_OOBDATA) flags |= MSG_OOB; + } else { + if ((flags & MSG_PEEK) == 0) { + KASSERT(so->so_rcv.sb_mb == m); + so->so_rcv.sb_mb = nextrecord; + SB_UPDATE_TAIL(&so->so_rcv); + } } + SBLASTRECORDCHK(&so->so_rcv, "soreceive 2"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 2"); + moff = 0; offset = 0; while (m && uio->uio_resid > 0 && error == 0) { @@ -992,6 +1028,8 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, * block interrupts again. */ if (mp == 0) { + SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove"); splx(s); error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio); s = splsoftnet(); @@ -1033,8 +1071,21 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, MFREE(m, so->so_rcv.sb_mb); m = so->so_rcv.sb_mb; } - if (m) + /* + * If m != NULL, we also know that + * so->so_rcv.sb_mb != NULL. + */ + KASSERT(so->so_rcv.sb_mb == m); + if (m) { m->m_nextpkt = nextrecord; + if (nextrecord == NULL) + so->so_rcv.sb_lastrecord = m; + } else { + so->so_rcv.sb_mb = nextrecord; + SB_UPDATE_TAIL(&so->so_rcv); + } + SBLASTRECORDCHK(&so->so_rcv, "soreceive 3"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 3"); } } else { if (flags & MSG_PEEK) @@ -1090,6 +1141,8 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, (struct mbuf *)(long)flags, (struct mbuf *)0, (struct proc *)0); + SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2"); error = sbwait(&so->so_rcv); if (error) { sbunlock(&so->so_rcv); @@ -1107,8 +1160,21 @@ soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio, (void) sbdroprecord(&so->so_rcv); } if ((flags & MSG_PEEK) == 0) { - if (m == 0) + if (m == 0) { + /* + * First part is an inline SB_UPDATE_TAIL(). Second + * part makes sure sb_lastrecord is up-to-date if + * there is still data in the socket buffer. + */ so->so_rcv.sb_mb = nextrecord; + if (so->so_rcv.sb_mb == NULL) { + so->so_rcv.sb_mbtail = NULL; + so->so_rcv.sb_lastrecord = NULL; + } else if (nextrecord->m_nextpkt == NULL) + so->so_rcv.sb_lastrecord = nextrecord; + } + SBLASTRECORDCHK(&so->so_rcv, "soreceive 4"); + SBLASTMBUFCHK(&so->so_rcv, "soreceive 4"); if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) (*pr->pr_usrreq)(so, PRU_RCVD, (struct mbuf *)0, (struct mbuf *)(long)flags, (struct mbuf *)0, diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c index 7a0cd92233af..34c0ef98af89 100644 --- a/sys/kern/uipc_socket2.c +++ b/sys/kern/uipc_socket2.c @@ -1,4 +1,4 @@ -/* $NetBSD: uipc_socket2.c,v 1.42 2001/11/12 15:25:33 lukem Exp $ */ +/* $NetBSD: uipc_socket2.c,v 1.43 2002/07/03 19:06:49 thorpej Exp $ */ /* * Copyright (c) 1982, 1986, 1988, 1990, 1993 @@ -36,7 +36,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.42 2001/11/12 15:25:33 lukem Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uipc_socket2.c,v 1.43 2002/07/03 19:06:49 thorpej Exp $"); #include #include @@ -425,6 +425,61 @@ sbrelease(struct sockbuf *sb) * or sbdroprecord() when the data is acknowledged by the peer. */ +#ifdef SOCKBUF_DEBUG +void +sblastrecordchk(struct sockbuf *sb, const char *where) +{ + struct mbuf *m = sb->sb_mb; + + while (m && m->m_nextpkt) + m = m->m_nextpkt; + + if (m != sb->sb_lastrecord) { + printf("sblastrecordchk: sb_mb %p sb_lastrecord %p last %p\n", + sb->sb_mb, sb->sb_lastrecord, m); + printf("packet chain:\n"); + for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) + printf("\t%p\n", m); + panic("sblastrecordchk from %s\n", where); + } +} + +void +sblastmbufchk(struct sockbuf *sb, const char *where) +{ + struct mbuf *m = sb->sb_mb; + struct mbuf *n; + + while (m && m->m_nextpkt) + m = m->m_nextpkt; + + while (m && m->m_next) + m = m->m_next; + + if (m != sb->sb_mbtail) { + printf("sblastmbufchk: sb_mb %p sb_mbtail %p last %p\n", + sb->sb_mb, sb->sb_mbtail, m); + printf("packet tree:\n"); + for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { + printf("\t"); + for (n = m; n != NULL; n = n->m_next) + printf("%p ", n); + printf("\n"); + } + panic("sblastmbufchk from %s", where); + } +} +#endif /* SOCKBUF_DEBUG */ + +#define SBLINKRECORD(sb, m0) \ +do { \ + if ((sb)->sb_lastrecord != NULL) \ + (sb)->sb_lastrecord->m_nextpkt = (m0); \ + else \ + (sb)->sb_mb = (m0); \ + (sb)->sb_lastrecord = (m0); \ +} while (/*CONSTCOND*/0) + /* * Append mbuf chain m to the last record in the * socket buffer sb. The additional space associated @@ -438,17 +493,50 @@ sbappend(struct sockbuf *sb, struct mbuf *m) if (m == 0) return; - if ((n = sb->sb_mb) != NULL) { - while (n->m_nextpkt) - n = n->m_nextpkt; + + SBLASTRECORDCHK(sb, "sbappend 1"); + + if ((n = sb->sb_lastrecord) != NULL) { + /* + * XXX Would like to simply use sb_mbtail here, but + * XXX I need to verify that I won't miss an EOR that + * XXX way. + */ do { if (n->m_flags & M_EOR) { sbappendrecord(sb, m); /* XXXXXX!!!! */ return; } } while (n->m_next && (n = n->m_next)); + } else { + /* + * If this is the first record in the socket buffer, it's + * also the last record. + */ + sb->sb_lastrecord = m; } sbcompress(sb, m, n); + SBLASTRECORDCHK(sb, "sbappend 2"); +} + +/* + * This version of sbappend() should only be used when the caller + * absolutely knows that there will never be more than one record + * in the socket buffer, that is, a stream protocol (such as TCP). + */ +void +sbappend_stream(struct sockbuf *sb, struct mbuf *m) +{ + + KDASSERT(m->m_nextpkt == NULL); + KASSERT(sb->sb_mb == sb->sb_lastrecord); + + SBLASTMBUFCHK(sb, __func__); + + sbcompress(sb, m, sb->sb_mbtail); + + sb->sb_lastrecord = sb->sb_mb; + SBLASTRECORDCHK(sb, __func__); } #ifdef SOCKBUF_DEBUG @@ -456,7 +544,7 @@ void sbcheck(struct sockbuf *sb) { struct mbuf *m; - int len, mbcnt; + u_long len, mbcnt; len = 0; mbcnt = 0; @@ -469,7 +557,7 @@ sbcheck(struct sockbuf *sb) panic("sbcheck nextpkt"); } if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { - printf("cc %d != %d || mbcnt %d != %d\n", len, sb->sb_cc, + printf("cc %lu != %lu || mbcnt %lu != %lu\n", len, sb->sb_cc, mbcnt, sb->sb_mbcnt); panic("sbcheck"); } @@ -487,18 +575,14 @@ sbappendrecord(struct sockbuf *sb, struct mbuf *m0) if (m0 == 0) return; - if ((m = sb->sb_mb) != NULL) - while (m->m_nextpkt) - m = m->m_nextpkt; + /* * Put the first mbuf on the queue. * Note this permits zero length records. */ sballoc(sb, m0); - if (m) - m->m_nextpkt = m0; - else - sb->sb_mb = m0; + SBLASTRECORDCHK(sb, "sbappendrecord 1"); + SBLINKRECORD(sb, m0); m = m0->m_next; m0->m_next = 0; if (m && (m0->m_flags & M_EOR)) { @@ -506,6 +590,7 @@ sbappendrecord(struct sockbuf *sb, struct mbuf *m0) m->m_flags |= M_EOR; } sbcompress(sb, m, m0); + SBLASTRECORDCHK(sb, "sbappendrecord 2"); } /* @@ -520,6 +605,9 @@ sbinsertoob(struct sockbuf *sb, struct mbuf *m0) if (m0 == 0) return; + + SBLASTRECORDCHK(sb, "sbinsertoob 1"); + for (mp = &sb->sb_mb; (m = *mp) != NULL; mp = &((*mp)->m_nextpkt)) { again: switch (m->m_type) { @@ -539,6 +627,10 @@ sbinsertoob(struct sockbuf *sb, struct mbuf *m0) */ sballoc(sb, m0); m0->m_nextpkt = *mp; + if (*mp == NULL) { + /* m0 is actually the new tail */ + sb->sb_lastrecord = m0; + } *mp = m0; m = m0->m_next; m0->m_next = 0; @@ -547,6 +639,7 @@ sbinsertoob(struct sockbuf *sb, struct mbuf *m0) m->m_flags |= M_EOR; } sbcompress(sb, m, m0); + SBLASTRECORDCHK(sb, "sbinsertoob 2"); } /* @@ -559,7 +652,7 @@ int sbappendaddr(struct sockbuf *sb, struct sockaddr *asa, struct mbuf *m0, struct mbuf *control) { - struct mbuf *m, *n; + struct mbuf *m, *n, *nlast; int space; space = asa->sa_len; @@ -592,21 +685,27 @@ sbappendaddr(struct sockbuf *sb, struct sockaddr *asa, struct mbuf *m0, else control = m0; m->m_next = control; - for (n = m; n; n = n->m_next) + + SBLASTRECORDCHK(sb, "sbappendaddr 1"); + + for (n = m; n->m_next != NULL; n = n->m_next) sballoc(sb, n); - if ((n = sb->sb_mb) != NULL) { - while (n->m_nextpkt) - n = n->m_nextpkt; - n->m_nextpkt = m; - } else - sb->sb_mb = m; + sballoc(sb, n); + nlast = n; + SBLINKRECORD(sb, m); + + sb->sb_mbtail = nlast; + SBLASTMBUFCHK(sb, "sbappendaddr"); + + SBLASTRECORDCHK(sb, "sbappendaddr 2"); + return (1); } int sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) { - struct mbuf *m, *n; + struct mbuf *m, *mlast, *n; int space; space = 0; @@ -623,14 +722,20 @@ sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) if (space > sbspace(sb)) return (0); n->m_next = m0; /* concatenate data to control */ - for (m = control; m; m = m->m_next) + + SBLASTRECORDCHK(sb, "sbappendcontrol 1"); + + for (m = control; m->m_next != NULL; m = m->m_next) sballoc(sb, m); - if ((n = sb->sb_mb) != NULL) { - while (n->m_nextpkt) - n = n->m_nextpkt; - n->m_nextpkt = control; - } else - sb->sb_mb = control; + sballoc(sb, m); + mlast = m; + SBLINKRECORD(sb, control); + + sb->sb_mbtail = mlast; + SBLASTMBUFCHK(sb, "sbappendcontrol"); + + SBLASTRECORDCHK(sb, "sbappendcontrol 2"); + return (1); } @@ -671,6 +776,7 @@ sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) n->m_next = m; else sb->sb_mb = m; + sb->sb_mbtail = m; sballoc(sb, m); n = m; m->m_flags &= ~M_EOR; @@ -683,6 +789,7 @@ sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) else printf("semi-panic: sbcompress\n"); } + SBLASTMBUFCHK(sb, __func__); } /* @@ -693,12 +800,15 @@ void sbflush(struct sockbuf *sb) { - if (sb->sb_flags & SB_LOCK) - panic("sbflush"); + KASSERT((sb->sb_flags & SB_LOCK) == 0); + while (sb->sb_mbcnt) sbdrop(sb, (int)sb->sb_cc); - if (sb->sb_cc || sb->sb_mb) - panic("sbflush 2"); + + KASSERT(sb->sb_cc == 0); + KASSERT(sb->sb_mb == NULL); + KASSERT(sb->sb_mbtail == NULL); + KASSERT(sb->sb_lastrecord == NULL); } /* @@ -739,6 +849,17 @@ sbdrop(struct sockbuf *sb, int len) m->m_nextpkt = next; } else sb->sb_mb = next; + /* + * First part is an inline SB_UPDATE_TAIL(). Second part + * makes sure sb_lastrecord is up-to-date if we dropped + * part of the last record. + */ + m = sb->sb_mb; + if (m == NULL) { + sb->sb_mbtail = NULL; + sb->sb_lastrecord = NULL; + } else if (m->m_nextpkt == NULL) + sb->sb_lastrecord = m; } /* @@ -758,6 +879,7 @@ sbdroprecord(struct sockbuf *sb) MFREE(m, mn); } while ((m = mn) != NULL); } + SB_UPDATE_TAIL(sb); } /* diff --git a/sys/netccitt/if_x25subr.c b/sys/netccitt/if_x25subr.c index 9b56343524b1..39f2f2276927 100644 --- a/sys/netccitt/if_x25subr.c +++ b/sys/netccitt/if_x25subr.c @@ -1,4 +1,4 @@ -/* $NetBSD: if_x25subr.c,v 1.28 2002/05/12 21:30:35 matt Exp $ */ +/* $NetBSD: if_x25subr.c,v 1.29 2002/07/03 19:06:52 thorpej Exp $ */ /* * Copyright (c) 1990, 1993 @@ -36,7 +36,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: if_x25subr.c,v 1.28 2002/05/12 21:30:35 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: if_x25subr.c,v 1.29 2002/07/03 19:06:52 thorpej Exp $"); #include "opt_inet.h" #include "opt_iso.h" @@ -770,7 +770,13 @@ pk_rtattach(so, m0) ((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long)) #define transfer_sockbuf(s, f, l) \ while ((m = (s)->sb_mb) != NULL) \ - {(s)->sb_mb = m->m_nextpkt; m->m_nextpkt = 0; sbfree((s), m); f;} + { \ + (s)->sb_mb = m->m_nextpkt; \ + SB_UPDATE_TAIL((s)); \ + m->m_nextpkt = 0; \ + sbfree((s), m); \ + f; \ + } if (rt) rt->rt_refcnt--; diff --git a/sys/netccitt/pk_output.c b/sys/netccitt/pk_output.c index e7b94a4c8a68..84b6210b6a5f 100644 --- a/sys/netccitt/pk_output.c +++ b/sys/netccitt/pk_output.c @@ -1,4 +1,4 @@ -/* $NetBSD: pk_output.c,v 1.16 2002/05/12 21:43:57 matt Exp $ */ +/* $NetBSD: pk_output.c,v 1.17 2002/07/03 19:06:53 thorpej Exp $ */ /* * Copyright (c) 1984 University of British Columbia. @@ -44,7 +44,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: pk_output.c,v 1.16 2002/05/12 21:43:57 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pk_output.c,v 1.17 2002/07/03 19:06:53 thorpej Exp $"); #include #include @@ -214,6 +214,7 @@ nextpk(lcp) return (NULL); sb->sb_mb = m->m_nextpkt; + SB_UPDATE_TAIL(sb); m->m_nextpkt = 0; for (n = m; n; n = n->m_next) sbfree(sb, n); diff --git a/sys/netccitt/pk_usrreq.c b/sys/netccitt/pk_usrreq.c index 2c1fe5353459..3cd20e663b98 100644 --- a/sys/netccitt/pk_usrreq.c +++ b/sys/netccitt/pk_usrreq.c @@ -1,4 +1,4 @@ -/* $NetBSD: pk_usrreq.c,v 1.21 2001/11/13 00:12:59 lukem Exp $ */ +/* $NetBSD: pk_usrreq.c,v 1.22 2002/07/03 19:06:54 thorpej Exp $ */ /* * Copyright (c) 1984 University of British Columbia. @@ -44,7 +44,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: pk_usrreq.c,v 1.21 2001/11/13 00:12:59 lukem Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pk_usrreq.c,v 1.22 2002/07/03 19:06:54 thorpej Exp $"); #include #include @@ -268,6 +268,7 @@ pk_usrreq(so, req, m, nam, control, p) if (n && n->m_type == MT_OOBDATA) { unsigned len = n->m_pkthdr.len; so->so_rcv.sb_mb = n->m_nextpkt; + SB_UPDATE_TAIL(&so->so_rcv); if (len != n->m_len && (n = m_pullup(n, len)) == 0) break; diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c index 0d00c88c1203..feb234220701 100644 --- a/sys/netinet/tcp_input.c +++ b/sys/netinet/tcp_input.c @@ -1,4 +1,4 @@ -/* $NetBSD: tcp_input.c,v 1.146 2002/06/30 22:40:35 thorpej Exp $ */ +/* $NetBSD: tcp_input.c,v 1.147 2002/07/03 19:06:50 thorpej Exp $ */ /* * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -152,7 +152,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: tcp_input.c,v 1.146 2002/06/30 22:40:35 thorpej Exp $"); +__KERNEL_RCSID(0, "$NetBSD: tcp_input.c,v 1.147 2002/07/03 19:06:50 thorpej Exp $"); #include "opt_inet.h" #include "opt_ipsec.h" @@ -664,7 +664,7 @@ present: if (so->so_state & SS_CANTRCVMORE) m_freem(q->ipqe_m); else - sbappend(&so->so_rcv, q->ipqe_m); + sbappend_stream(&so->so_rcv, q->ipqe_m); pool_put(&ipqent_pool, q); sorwakeup(so); return (pkt_flags); @@ -1524,7 +1524,7 @@ after_listen: * to socket buffer. */ m_adj(m, toff + off); - sbappend(&so->so_rcv, m); + sbappend_stream(&so->so_rcv, m); sorwakeup(so); TCP_SETUP_ACK(tp, th); if (tp->t_flags & TF_ACKNOW) @@ -2263,7 +2263,7 @@ dodata: /* XXX */ tcpstat.tcps_rcvbyte += tlen; ND6_HINT(tp); m_adj(m, hdroptlen); - sbappend(&(so)->so_rcv, m); + sbappend_stream(&(so)->so_rcv, m); sorwakeup(so); } else { m_adj(m, hdroptlen); diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 5a56431561b6..3398fbeed762 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -1,4 +1,4 @@ -/* $NetBSD: tcp_usrreq.c,v 1.71 2002/06/09 16:33:44 itojun Exp $ */ +/* $NetBSD: tcp_usrreq.c,v 1.72 2002/07/03 19:06:52 thorpej Exp $ */ /* * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -102,7 +102,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: tcp_usrreq.c,v 1.71 2002/06/09 16:33:44 itojun Exp $"); +__KERNEL_RCSID(0, "$NetBSD: tcp_usrreq.c,v 1.72 2002/07/03 19:06:52 thorpej Exp $"); #include "opt_inet.h" #include "opt_ipsec.h" @@ -508,7 +508,7 @@ tcp_usrreq(so, req, m, nam, control, p) error = EINVAL; break; } - sbappend(&so->so_snd, m); + sbappend_stream(&so->so_snd, m); error = tcp_output(tp); break; @@ -564,7 +564,7 @@ tcp_usrreq(so, req, m, nam, control, p) * of data past the urgent section. * Otherwise, snd_up should be one lower. */ - sbappend(&so->so_snd, m); + sbappend_stream(&so->so_snd, m); tp->snd_up = tp->snd_una + so->so_snd.sb_cc; tp->t_force = 1; error = tcp_output(tp); diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h index 7155113cf10b..e7cae44b16c0 100644 --- a/sys/sys/socketvar.h +++ b/sys/sys/socketvar.h @@ -1,4 +1,4 @@ -/* $NetBSD: socketvar.h,v 1.51 2002/05/02 17:55:52 thorpej Exp $ */ +/* $NetBSD: socketvar.h,v 1.52 2002/07/03 19:06:54 thorpej Exp $ */ /*- * Copyright (c) 1982, 1986, 1990, 1993 @@ -93,6 +93,9 @@ struct socket { u_long sb_mbmax; /* max chars of mbufs to use */ long sb_lowat; /* low water mark */ struct mbuf *sb_mb; /* the mbuf chain */ + struct mbuf *sb_mbtail; /* the last mbuf in the chain */ + struct mbuf *sb_lastrecord;/* first mbuf of last record in + socket buffer */ struct selinfo sb_sel; /* process selecting read/write */ short sb_flags; /* flags, see below */ short sb_timeo; /* timeout for read/write */ @@ -125,6 +128,14 @@ struct socket { struct mbuf *so_pendfree; /* loaned-page mbufs w/ frees pending */ }; +#define SB_UPDATE_TAIL(sb) \ +do { \ + if ((sb)->sb_mb == NULL) { \ + (sb)->sb_mbtail = NULL; \ + (sb)->sb_lastrecord = NULL; \ + } \ +} while (/*CONSTCOND*/0) + /* * Socket state bits. */ @@ -266,6 +277,7 @@ int uipc_usrreq(struct socket *, int , struct mbuf *, struct mbuf *, struct mbuf *, struct proc *); int uipc_ctloutput(int, struct socket *, int, int, struct mbuf **); void sbappend(struct sockbuf *sb, struct mbuf *m); +void sbappend_stream(struct sockbuf *sb, struct mbuf *m); int sbappendaddr(struct sockbuf *sb, struct sockaddr *asa, struct mbuf *m0, struct mbuf *control); int sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, @@ -320,6 +332,17 @@ int sockargs(struct mbuf **, const void *, size_t, int); int sendit(struct proc *, int, struct msghdr *, int, register_t *); int recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *); +#ifdef SOCKBUF_DEBUG +void sblastrecordchk(struct sockbuf *, const char *); +#define SBLASTRECORDCHK(sb, where) sblastrecordchk((sb), (where)) + +void sblastmbufchk(struct sockbuf *, const char *); +#define SBLASTMBUFCHK(sb, where) sblastmbufchk((sb), (where)) +#else +#define SBLASTRECORDCHK(sb, where) /* nothing */ +#define SBLASTMBUFCHK(sb, where) /* nothing */ +#endif /* SOCKBUF_DEBUG */ + #endif /* _KERNEL */ #endif /* !_SYS_SOCKETVAR_H_ */