Add optional mmap(2)/write(2) support for binary file transfer.

The default is read(2)/write(2).  Note that the sosend_loan needs
some more work for better performance when a file isn't cached.
This commit is contained in:
enami 2002-05-30 00:24:47 +00:00
parent e67961b545
commit 260e9f55a7
4 changed files with 343 additions and 72 deletions

View File

@ -1,4 +1,4 @@
/* $NetBSD: conf.c,v 1.46 2001/12/04 13:54:12 lukem Exp $ */
/* $NetBSD: conf.c,v 1.47 2002/05/30 00:24:47 enami Exp $ */
/*-
* Copyright (c) 1997-2001 The NetBSD Foundation, Inc.
@ -38,7 +38,7 @@
#include <sys/cdefs.h>
#ifndef lint
__RCSID("$NetBSD: conf.c,v 1.46 2001/12/04 13:54:12 lukem Exp $");
__RCSID("$NetBSD: conf.c,v 1.47 2002/05/30 00:24:47 enami Exp $");
#endif /* not lint */
#include <sys/types.h>
@ -119,6 +119,11 @@ init_curclass(void)
curclass.timeout = DEFAULT_TIMEOUT;
/* curclass.type is set elsewhere */
curclass.umask = DEFAULT_UMASK;
curclass.mmapsize = 0;
curclass.readsize = 0;
curclass.writesize = 0;
curclass.sendbufsize = 0;
curclass.sendlowat = 0;
CURCLASS_FLAGS_SET(checkportcmd);
CURCLASS_FLAGS_CLR(denyquick);
@ -398,6 +403,71 @@ parse_conf(const char *findclass)
}
curclass.maxtimeout = timeout;
} else if (strcasecmp(word, "mmapsize") == 0) {
curclass.mmapsize = 0;
if (none || EMPTYSTR(arg))
continue;
llval = strsuftoll(arg);
if (llval == -1) {
syslog(LOG_WARNING,
"%s line %d: invalid mmapsize %s",
infile, (int)line, arg);
continue;
}
curclass.mmapsize = llval;
} else if (strcasecmp(word, "readsize") == 0) {
curclass.readsize = 0;
if (none || EMPTYSTR(arg))
continue;
llval = strsuftoll(arg);
if (llval == -1) {
syslog(LOG_WARNING,
"%s line %d: invalid readsize %s",
infile, (int)line, arg);
continue;
}
curclass.readsize = llval;
} else if (strcasecmp(word, "writesize") == 0) {
curclass.writesize = 0;
if (none || EMPTYSTR(arg))
continue;
llval = strsuftoll(arg);
if (llval == -1) {
syslog(LOG_WARNING,
"%s line %d: invalid writesize %s",
infile, (int)line, arg);
continue;
}
curclass.writesize = llval;
} else if (strcasecmp(word, "sendbufsize") == 0) {
curclass.sendbufsize = 0;
if (none || EMPTYSTR(arg))
continue;
llval = strsuftoll(arg);
if (llval == -1) {
syslog(LOG_WARNING,
"%s line %d: invalid sendbufsize %s",
infile, (int)line, arg);
continue;
}
curclass.sendbufsize = llval;
} else if (strcasecmp(word, "sendlowat") == 0) {
curclass.sendlowat = 0;
if (none || EMPTYSTR(arg))
continue;
llval = strsuftoll(arg);
if (llval == -1) {
syslog(LOG_WARNING,
"%s line %d: invalid sendlowat %s",
infile, (int)line, arg);
continue;
}
curclass.sendlowat = llval;
} else if (strcasecmp(word, "modify") == 0) {
CONF_FLAG(modify);

View File

@ -1,4 +1,4 @@
/* $NetBSD: extern.h,v 1.43 2001/12/04 13:54:12 lukem Exp $ */
/* $NetBSD: extern.h,v 1.44 2002/05/30 00:24:47 enami Exp $ */
/*-
* Copyright (c) 1992, 1993
@ -276,6 +276,11 @@ struct ftpclass {
unsigned int timeout; /* Default timeout */
class_ft type; /* Class type */
mode_t umask; /* Umask to use */
LLT mmapsize; /* mmap window size */
LLT readsize; /* data read size */
LLT writesize; /* data write size */
LLT sendbufsize; /* SO_SNDBUF size */
LLT sendlowat; /* SO_SNDLOWAT size */
};
extern void ftp_loop(void) __attribute__ ((noreturn));

View File

@ -1,4 +1,4 @@
/* $NetBSD: ftpd.c,v 1.138 2002/02/11 11:45:07 lukem Exp $ */
/* $NetBSD: ftpd.c,v 1.139 2002/05/30 00:24:47 enami Exp $ */
/*
* Copyright (c) 1997-2001 The NetBSD Foundation, Inc.
@ -109,7 +109,7 @@ __COPYRIGHT(
#if 0
static char sccsid[] = "@(#)ftpd.c 8.5 (Berkeley) 4/28/95";
#else
__RCSID("$NetBSD: ftpd.c,v 1.138 2002/02/11 11:45:07 lukem Exp $");
__RCSID("$NetBSD: ftpd.c,v 1.139 2002/05/30 00:24:47 enami Exp $");
#endif
#endif /* not lint */
@ -121,6 +121,8 @@ __RCSID("$NetBSD: ftpd.c,v 1.138 2002/02/11 11:45:07 lukem Exp $");
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
@ -208,6 +210,13 @@ int epsvall = 0;
int swaitmax = SWAITMAX;
int swaitint = SWAITINT;
enum send_status {
SS_SUCCESS,
SS_NO_TRANSFER, /* no transfer made yet */
SS_FILE_ERROR, /* file read error */
SS_DATA_ERROR /* data send error */
};
static int bind_pasv_addr(void);
static int checkuser(const char *, const char *, int, int, char **);
static int checkaccess(const char *);
@ -219,8 +228,15 @@ static void logremotehost(struct sockinet *);
static void lostconn(int);
static void myoob(int);
static int receive_data(FILE *, FILE *);
static int send_data(FILE *, FILE *, off_t, int);
static int send_data(FILE *, FILE *, const struct stat *, int);
static struct passwd *sgetpwnam(const char *);
static int write_data(int, char *, size_t, off_t *, struct timeval *,
int);
static enum send_status
send_data_with_read(int, int, const struct stat *, int);
static enum send_status
send_data_with_mmap(int, int, const struct stat *, int);
static void logrusage(const struct rusage *, const struct rusage *);
int main(int, char *[]);
@ -1206,6 +1222,7 @@ retrieve(char *argv[], const char *name)
int (*closefunc)(FILE *) = NULL;
int log, sendrv, closerv, stderrfd, isconversion, isdata, isls;
struct timeval start, finish, td, *tdp;
struct rusage rusage_before, rusage_after;
const char *dispname;
char *error;
@ -1286,15 +1303,20 @@ retrieve(char *argv[], const char *name)
if (dout == NULL)
goto done;
(void)getrusage(RUSAGE_SELF, &rusage_before);
(void)gettimeofday(&start, NULL);
sendrv = send_data(fin, dout, st.st_blksize, isdata);
sendrv = send_data(fin, dout, &st, isdata);
(void)gettimeofday(&finish, NULL);
(void)getrusage(RUSAGE_SELF, &rusage_after);
closedataconn(dout); /* close now to affect timing stats */
timersub(&finish, &start, &td);
tdp = &td;
done:
if (log)
if (log) {
logxfer("get", byte_count, name, NULL, tdp, error);
if (tdp != NULL)
logrusage(&rusage_before, &rusage_after);
}
closerv = (*closefunc)(fin);
if (sendrv == 0) {
FILE *errf;
@ -1585,21 +1607,173 @@ closedataconn(FILE *fd)
pdata = -1;
}
int
write_data(int fd, char *buf, size_t size, off_t *bufrem,
struct timeval *then, int isdata)
{
struct timeval now, td;
ssize_t c;
while (size > 0) {
c = size;
if (curclass.writesize) {
if (curclass.writesize < c)
c = curclass.writesize;
}
if (curclass.rateget) {
if (*bufrem < c)
c = *bufrem;
}
(void) alarm(curclass.timeout);
c = write(fd, buf, c);
if (c <= 0)
return (1);
buf += c;
size -= c;
byte_count += c;
if (isdata) {
total_data_out += c;
total_data += c;
}
total_bytes_out += c;
total_bytes += c;
if (curclass.rateget) {
*bufrem -= c;
if (*bufrem == 0) {
(void)gettimeofday(&now, NULL);
timersub(&now, then, &td);
if (td.tv_sec == 0) {
usleep(1000000 - td.tv_usec);
(void)gettimeofday(then, NULL);
} else
*then = now;
*bufrem = curclass.rateget;
}
}
}
return (0);
}
static enum send_status
send_data_with_read(int filefd, int netfd, const struct stat *st, int isdata)
{
struct timeval then;
off_t bufrem;
size_t readsize;
char *buf;
int c, error;
if (curclass.readsize)
readsize = curclass.readsize;
else
readsize = (size_t)st->st_blksize;
if ((buf = malloc(readsize)) == NULL) {
perror_reply(451, "Local resource failure: malloc");
return (SS_NO_TRANSFER);
}
if (curclass.rateget) {
bufrem = curclass.rateget;
(void)gettimeofday(&then, NULL);
}
while (1) {
(void) alarm(curclass.timeout);
c = read(filefd, buf, readsize);
if (c == 0)
error = SS_SUCCESS;
else if (c < 0)
error = SS_FILE_ERROR;
else if (write_data(netfd, buf, c, &bufrem, &then, isdata))
error = SS_DATA_ERROR;
else
continue;
free(buf);
return (error);
}
}
static enum send_status
send_data_with_mmap(int filefd, int netfd, const struct stat *st, int isdata)
{
struct timeval then;
off_t bufrem, filesize, off, origoff;
size_t mapsize, winsize;
int error, sendbufsize, sendlowat;
void *win;
if (curclass.sendbufsize) {
sendbufsize = curclass.sendbufsize;
if (setsockopt(netfd, SOL_SOCKET, SO_SNDBUF,
&sendbufsize, sizeof(int)) == -1)
syslog(LOG_WARNING, "setsockopt(SO_SNDBUF, %d): %m",
sendbufsize);
}
if (curclass.sendlowat) {
sendlowat = curclass.sendlowat;
if (setsockopt(netfd, SOL_SOCKET, SO_SNDLOWAT,
&sendlowat, sizeof(int)) == -1)
syslog(LOG_WARNING, "setsockopt(SO_SNDLOWAT, %d): %m",
sendlowat);
}
winsize = curclass.mmapsize;
filesize = st->st_size;
if (debug)
syslog(LOG_INFO, "mmapsize = %ld, writesize = %ld",
(long)winsize, (long)curclass.writesize);
if (winsize == 0)
goto try_read;
off = lseek(filefd, (off_t)0, SEEK_CUR);
if (off == -1)
goto try_read;
origoff = off;
if (curclass.rateget) {
bufrem = curclass.rateget;
(void)gettimeofday(&then, NULL);
}
while (1) {
mapsize = MIN(filesize - off, winsize);
if (mapsize == 0)
break;
win = mmap(NULL, mapsize, PROT_READ,
MAP_FILE|MAP_SHARED, filefd, off);
if (win == MAP_FAILED) {
if (off == origoff)
goto try_read;
return (SS_FILE_ERROR);
}
(void) madvise(win, mapsize, MADV_SEQUENTIAL);
error = write_data(netfd, win, mapsize, &bufrem, &then,
isdata);
(void) madvise(win, mapsize, MADV_DONTNEED);
munmap(win, mapsize);
if (error)
return (SS_DATA_ERROR);
off += mapsize;
}
return (SS_SUCCESS);
try_read:
return (send_data_with_read(filefd, netfd, st, isdata));
}
/*
* Tranfer the contents of "instr" to "outstr" peer using the appropriate
* encapsulation of the data subject * to Mode, Structure, and Type.
* encapsulation of the data subject to Mode, Structure, and Type.
*
* NB: Form isn't handled.
*/
static int
send_data(FILE *instr, FILE *outstr, off_t blksize, int isdata)
send_data(FILE *instr, FILE *outstr, const struct stat *st, int isdata)
{
int c, filefd, netfd, rval;
char *buf;
transflag = 1;
rval = -1;
buf = NULL;
if (setjmp(urgcatch))
goto cleanup_send_data;
@ -1642,66 +1816,22 @@ send_data(FILE *instr, FILE *outstr, off_t blksize, int isdata)
case TYPE_I:
case TYPE_L:
if ((buf = malloc((size_t)blksize)) == NULL) {
perror_reply(451, "Local resource failure: malloc");
goto cleanup_send_data;
}
filefd = fileno(instr);
netfd = fileno(outstr);
(void) alarm(curclass.timeout);
if (curclass.rateget) {
while (1) {
int d;
struct timeval then, now, td;
off_t bufrem;
char *bufp;
switch (send_data_with_mmap(filefd, netfd, st, isdata)) {
(void)gettimeofday(&then, NULL);
errno = c = d = 0;
bufrem = curclass.rateget;
while (bufrem > 0) {
if ((c = read(filefd, buf,
MIN(blksize, bufrem))) <= 0)
goto senddone;
(void) alarm(curclass.timeout);
bufrem -= c;
byte_count += c;
if (isdata) {
total_data_out += c;
total_data += c;
}
total_bytes_out += c;
total_bytes += c;
for (bufp = buf; c > 0;
c -= d, bufp += d)
if ((d =
write(netfd, bufp, c)) <= 0)
break;
if (d < 0)
goto data_err;
}
(void)gettimeofday(&now, NULL);
timersub(&now, &then, &td);
if (td.tv_sec == 0)
usleep(1000000 - td.tv_usec);
}
} else {
while ((c = read(filefd, buf, (size_t)blksize)) > 0) {
if (write(netfd, buf, c) != c)
goto data_err;
(void) alarm(curclass.timeout);
byte_count += c;
if (isdata) {
total_data_out += c;
total_data += c;
}
total_bytes_out += c;
total_bytes += c;
}
}
senddone:
if (c < 0)
case SS_SUCCESS:
break;
case SS_NO_TRANSFER:
goto cleanup_send_data;
case SS_FILE_ERROR:
goto file_err;
case SS_DATA_ERROR:
goto data_err;
}
rval = 0;
goto cleanup_send_data;
@ -1723,8 +1853,6 @@ send_data(FILE *instr, FILE *outstr, off_t blksize, int isdata)
cleanup_send_data:
(void) alarm(0);
transflag = 0;
if (buf)
free(buf);
if (isdata) {
total_files_out++;
total_files++;
@ -2851,7 +2979,7 @@ conffilename(const char *s)
*/
void
logxfer(const char *command, off_t bytes, const char *file1, const char *file2,
const struct timeval *elapsed, const char *error)
const struct timeval *elapsed, const char *error)
{
char buf[MAXPATHLEN * 2 + 100], realfile[MAXPATHLEN];
const char *r1, *r2;
@ -2937,6 +3065,31 @@ logxfer(const char *command, off_t bytes, const char *file1, const char *file2,
);
}
/*
* Log the resource usage.
*
* XXX: more resource usage to logging?
*/
void
logrusage(const struct rusage *rusage_before,
const struct rusage *rusage_after)
{
struct timeval usrtime, systime;
if (logging <= 1)
return;
timersub(&rusage_after->ru_utime, &rusage_before->ru_utime, &usrtime);
timersub(&rusage_after->ru_stime, &rusage_before->ru_stime, &systime);
syslog(LOG_INFO, "%ld.%.03du %ld.%.03ds %ld+%ldio %ldpf+%ldw",
usrtime.tv_sec, (int)(usrtime.tv_usec / 1000),
systime.tv_sec, (int)(systime.tv_usec / 1000),
rusage_after->ru_inblock - rusage_before->ru_inblock,
rusage_after->ru_oublock - rusage_before->ru_oublock,
rusage_after->ru_majflt - rusage_before->ru_majflt,
rusage_after->ru_nswap - rusage_before->ru_nswap);
}
/*
* Determine if `password' is valid for user given in `pw'.
* Returns 2 if password expired, 1 if otherwise failed, 0 if ok

View File

@ -1,4 +1,4 @@
.\" $NetBSD: ftpd.conf.5,v 1.19 2002/01/15 02:20:50 wiz Exp $
.\" $NetBSD: ftpd.conf.5,v 1.20 2002/05/30 00:24:47 enami Exp $
.\"
.\" Copyright (c) 1997-2001 The NetBSD Foundation, Inc.
.\" All rights reserved.
@ -376,6 +376,17 @@ is
or
.Ar time
is not specified, set to default of 2 hours.
.It Sy mmapsize Ar class Ar size
Set the size of sliding window to map a file using
.Xr mmap 2 .
If zero,
.Xr ftpd 8
will use
.Xr read 2
instead. The default is zero.
An optional suffix may be provided as per
.Sy rateget .
This option affects only for binary transfer.
.It Sy modify Ar class Op Sy off
If
.Ar class
@ -510,6 +521,14 @@ If
is
.Dq none
or no arguments are given, disable this.
.It Sy readsize Ar class Ar size
Set the size of read buffer to
.Xr read 2
a file.
The default is file system block size.
An optional suffix may be provided as per
.Sy rateget .
This option affects only for binary transfer.
.It Sy sanenames Ar class Op Sy off
If
.Ar class
@ -523,6 +542,18 @@ Otherwise, only permit file names which don't start with a
.Sq \&.
and only comprise of characters from the set
.Dq [-+,._A-Za-z0-9] .
.It Sy sendbufsize Ar class Ar size
Set the size of socket send buffer.
An optional suffix may be provided as per
.Sy rateget .
The default is zero and system default value will be used.
This option affects only for binary transfer.
.It Sy sendlowat Ar class Ar size
Set the low water mark of socket send buffer.
An optional suffix may be provided as per
.Sy rateget .
The default is zero and system default value will be used.
This option affects only for binary transfer.
.It Sy template Ar class Op Ar refclass
Define
.Ar refclass
@ -586,6 +617,18 @@ as well as the modify commands:
and
.Sy UMASK .
Otherwise, enable them.
.It Sy writesize Ar class Ar size
Limit the number of bytes to
.Xr write 2
at a time. The default is zero and
whole data available as a resul of
.Xr mmap 2
or
.Xr read 2
will be written at a time.
An optional suffix may be provided as per
.Sy rateget .
This option affects only for binary transfer.
.El
.Sh DEFAULTS
The following defaults are used: