Implement "superreaddir". This issues a getattr for all the
directory entries already in readdir and caches the results instead of waiting for each individial getattr from the kernel. For high-latency links the difference in "ls -l" is quite astounding and even on my lan "ls -lR" is faster than for nfs in a normal directory hierarchy (i.e. not one artifically setup to have thousands of files per directory). TODO: implement some sort of bandwidth/latency measurement in the code and enable or disable this option based on than information (and a command-line flag).
This commit is contained in:
pooka
parent
c653ab50ef
commit
306e0025cf
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: node.c,v 1.5 2007/01/15 00:42:21 pooka Exp $ */
|
||||
/* $NetBSD: node.c,v 1.6 2007/02/09 23:36:17 pooka Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2006 Antti Kantee. All Rights Reserved.
|
||||
|
|
@ -30,7 +30,7 @@
|
|||
|
||||
#include <sys/cdefs.h>
|
||||
#ifndef lint
|
||||
__RCSID("$NetBSD: node.c,v 1.5 2007/01/15 00:42:21 pooka Exp $");
|
||||
__RCSID("$NetBSD: node.c,v 1.6 2007/02/09 23:36:17 pooka Exp $");
|
||||
#endif /* !lint */
|
||||
|
||||
#include <assert.h>
|
||||
|
|
@ -67,8 +67,9 @@ psshfs_node_lookup(struct puffs_cc *pcc, void *opc, void **newnode,
|
|||
}
|
||||
|
||||
pd = lookup(psn_dir->dir, psn_dir->dentnext, pcn->pcn_name);
|
||||
if (!pd)
|
||||
if (!pd) {
|
||||
return ENOENT;
|
||||
}
|
||||
|
||||
if (pd->entry)
|
||||
pn = pd->entry;
|
||||
|
|
@ -93,8 +94,7 @@ psshfs_node_getattr(struct puffs_cc *pcc, void *opc, struct vattr *vap,
|
|||
|
||||
rv = 0;
|
||||
|
||||
/* XXX: expire by time */
|
||||
if (!psn->hasvattr) {
|
||||
if ((time(NULL) - psn->attrread) >= PSSHFS_REFRESHIVAL) {
|
||||
psbuf_req_str(pb, SSH_FXP_LSTAT, reqid, PNPATH(pn));
|
||||
pssh_outbuf_enqueue(pctx, pb, pcc, reqid);
|
||||
puffs_cc_yield(pcc);
|
||||
|
|
@ -103,11 +103,24 @@ psshfs_node_getattr(struct puffs_cc *pcc, void *opc, struct vattr *vap,
|
|||
if (rv)
|
||||
goto out;
|
||||
|
||||
#if 0
|
||||
/*
|
||||
* check if the file was modified from below us
|
||||
*
|
||||
* XXX: what's the right place(s) to do this?
|
||||
* XXX2: resolution only per second, since sftp doesn't
|
||||
* support nanoseconds
|
||||
*/
|
||||
if (psn->attrread)
|
||||
if (pn->pn_va.va_mtime.tv_sec != va.va_mtime.tv_sec)
|
||||
puffs_inval_pagecache_node(pu, opc);
|
||||
#endif
|
||||
|
||||
puffs_setvattr(&pn->pn_va, &va);
|
||||
psn->attrread = time(NULL);
|
||||
}
|
||||
|
||||
memcpy(vap, &pn->pn_va, sizeof(struct vattr));
|
||||
psn->hasvattr = 1;
|
||||
|
||||
out:
|
||||
PSSHFSRETURN(rv);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: psshfs.c,v 1.6 2007/01/20 14:37:48 pooka Exp $ */
|
||||
/* $NetBSD: psshfs.c,v 1.7 2007/02/09 23:36:17 pooka Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2006 Antti Kantee. All Rights Reserved.
|
||||
|
|
@ -57,7 +57,7 @@
|
|||
|
||||
#include <sys/cdefs.h>
|
||||
#ifndef lint
|
||||
__RCSID("$NetBSD: psshfs.c,v 1.6 2007/01/20 14:37:48 pooka Exp $");
|
||||
__RCSID("$NetBSD: psshfs.c,v 1.7 2007/02/09 23:36:17 pooka Exp $");
|
||||
#endif /* !lint */
|
||||
|
||||
#include <sys/types.h>
|
||||
|
|
@ -128,6 +128,11 @@ main(int argc, char *argv[])
|
|||
argc -= optind;
|
||||
argv += optind;
|
||||
|
||||
#if 0
|
||||
/* XXX: noatime is mandatory for now */
|
||||
mntflags |= MNT_NOATIME;
|
||||
#endif
|
||||
|
||||
if (pflags & PUFFS_FLAG_OPDUMP)
|
||||
detach = 0;
|
||||
|
||||
|
|
@ -198,6 +203,9 @@ main(int argc, char *argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* enqueue buffer to be handled with cc
|
||||
*/
|
||||
void
|
||||
pssh_outbuf_enqueue(struct psshfs_ctx *pctx, struct psbuf *pb,
|
||||
struct puffs_cc *pcc, uint32_t reqid)
|
||||
|
|
@ -205,14 +213,26 @@ pssh_outbuf_enqueue(struct psshfs_ctx *pctx, struct psbuf *pb,
|
|||
|
||||
pb->psr.reqid = reqid;
|
||||
pb->psr.pcc = pcc;
|
||||
pb->psr.func = NULL;
|
||||
pb->psr.arg = NULL;
|
||||
TAILQ_INSERT_TAIL(&pctx->outbufq, pb, psr.entries);
|
||||
}
|
||||
|
||||
/*
|
||||
* enqueue buffer to be handled with "f". "f" must not block.
|
||||
* gives up struct psbuf ownership.
|
||||
*/
|
||||
void
|
||||
psshreq_put(struct psshfs_ctx *pctx, struct psbuf *pb)
|
||||
pssh_outbuf_enqueue_nocc(struct psshfs_ctx *pctx, struct psbuf *pb,
|
||||
void (*f)(struct psshfs_ctx *, struct psbuf *, void *), void *arg,
|
||||
uint32_t reqid)
|
||||
{
|
||||
|
||||
TAILQ_INSERT_TAIL(&pctx->req_queue, pb, psr.entries);
|
||||
pb->psr.reqid = reqid;
|
||||
pb->psr.pcc = NULL;
|
||||
pb->psr.func = f;
|
||||
pb->psr.arg = arg;
|
||||
TAILQ_INSERT_TAIL(&pctx->outbufq, pb, psr.entries);
|
||||
}
|
||||
|
||||
struct psbuf *
|
||||
|
|
@ -242,19 +262,24 @@ handlebuf(struct psshfs_ctx *pctx, struct psbuf *datapb,
|
|||
/* is this something we are expecting? */
|
||||
pb = psshreq_get(pctx, datapb->reqid);
|
||||
|
||||
if (pb) {
|
||||
/* keep psreq clean, xxx uknow */
|
||||
psrtmp = pb->psr;
|
||||
*pb = *datapb;
|
||||
pb->psr = psrtmp;
|
||||
free(datapb);
|
||||
|
||||
puffs_docc(pb->psr.pcc, ppr);
|
||||
if (pb == NULL) {
|
||||
printf("invalid server request response %d\n", datapb->reqid);
|
||||
psbuf_destroy(datapb);
|
||||
return;
|
||||
}
|
||||
|
||||
printf("invalid server request response %d\n", datapb->reqid);
|
||||
psbuf_destroy(datapb);
|
||||
/* keep psreq clean, xxx uknow */
|
||||
psrtmp = pb->psr;
|
||||
*pb = *datapb;
|
||||
pb->psr = psrtmp;
|
||||
free(datapb);
|
||||
|
||||
assert((pb->psr.pcc && pb->psr.func) == 0);
|
||||
if (pb->psr.pcc) {
|
||||
puffs_docc(pb->psr.pcc, ppr);
|
||||
} else {
|
||||
pb->psr.func(pctx, pb, pb->psr.arg);
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: psshfs.h,v 1.3 2007/01/15 00:42:21 pooka Exp $ */
|
||||
/* $NetBSD: psshfs.h,v 1.4 2007/02/09 23:36:17 pooka Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2006 Antti Kantee. All Rights Reserved.
|
||||
|
|
@ -47,7 +47,10 @@
|
|||
* Refresh directories every n seconds as indicated by the following macro.
|
||||
* Note that local changes will still be visible immediately.
|
||||
*/
|
||||
#define PSSHFS_REFRESHIVAL 10
|
||||
#define PSSHFS_REFRESHIVAL 30
|
||||
|
||||
/* warm getattr cache in readdir */
|
||||
#define SUPERREADDIR
|
||||
|
||||
PUFFSOP_PROTOS(psshfs);
|
||||
|
||||
|
|
@ -69,6 +72,7 @@ struct psshfs_dir {
|
|||
|
||||
char *entryname;
|
||||
struct vattr va;
|
||||
time_t attrread;
|
||||
};
|
||||
|
||||
struct psshfs_node {
|
||||
|
|
@ -78,11 +82,12 @@ struct psshfs_node {
|
|||
size_t denttot;
|
||||
size_t dentnext;
|
||||
time_t dentread;
|
||||
int hasvattr;
|
||||
|
||||
int childcount;
|
||||
|
||||
time_t attrread;
|
||||
};
|
||||
|
||||
struct psshfs_ctx;
|
||||
/*
|
||||
* XXX: urgh
|
||||
*
|
||||
|
|
@ -100,8 +105,16 @@ struct psshfs_node {
|
|||
#define PSB_IN 1
|
||||
struct psbuf {
|
||||
struct psreq {
|
||||
struct puffs_cc *pcc;
|
||||
uint32_t reqid;
|
||||
|
||||
/* either ... */
|
||||
struct puffs_cc *pcc;
|
||||
|
||||
/* ... or */
|
||||
void (*func)(struct psshfs_ctx *, struct psbuf *, void *);
|
||||
void *arg;
|
||||
/* no union, we'd need a "which" flag */
|
||||
|
||||
TAILQ_ENTRY(psbuf) entries;
|
||||
} psr;
|
||||
|
||||
|
|
@ -151,7 +164,10 @@ int psbuf_write(struct psshfs_ctx *, struct psbuf *);
|
|||
|
||||
void pssh_outbuf_enqueue(struct psshfs_ctx *, struct psbuf *,
|
||||
struct puffs_cc *, uint32_t);
|
||||
void psshreq_put(struct psshfs_ctx *, struct psbuf *);
|
||||
void pssh_outbuf_enqueue_nocc(struct psshfs_ctx *, struct psbuf *,
|
||||
void (*f)(struct psshfs_ctx *,
|
||||
struct psbuf *, void *),
|
||||
void *, uint32_t);
|
||||
struct psbuf *psshreq_get(struct psshfs_ctx *, uint32_t);
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: subr.c,v 1.7 2007/01/15 00:42:21 pooka Exp $ */
|
||||
/* $NetBSD: subr.c,v 1.8 2007/02/09 23:36:17 pooka Exp $ */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2006 Antti Kantee. All Rights Reserved.
|
||||
|
|
@ -30,7 +30,7 @@
|
|||
|
||||
#include <sys/cdefs.h>
|
||||
#ifndef lint
|
||||
__RCSID("$NetBSD: subr.c,v 1.7 2007/01/15 00:42:21 pooka Exp $");
|
||||
__RCSID("$NetBSD: subr.c,v 1.8 2007/02/09 23:36:17 pooka Exp $");
|
||||
#endif /* !lint */
|
||||
|
||||
#include <assert.h>
|
||||
|
|
@ -84,6 +84,70 @@ lookup(struct psshfs_dir *basedir, size_t ndir, const char *name)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
#ifdef SUPERREADDIR
|
||||
struct readdirattr {
|
||||
struct psshfs_node *psn;
|
||||
int idx;
|
||||
char entryname[MAXPATHLEN+1];
|
||||
};
|
||||
|
||||
static void
|
||||
readdir_getattr_resp(struct psshfs_ctx *pctx, struct psbuf *pb, void *arg)
|
||||
{
|
||||
struct readdirattr *rda = arg;
|
||||
struct psshfs_node *psn = rda->psn;
|
||||
struct psshfs_dir *pdir;
|
||||
struct vattr va;
|
||||
|
||||
pdir = lookup(psn->dir, psn->denttot, rda->entryname);
|
||||
if (!pdir)
|
||||
goto out;
|
||||
|
||||
if (psbuf_expect_attrs(pb, &va))
|
||||
goto out;
|
||||
|
||||
if (pdir->entry) {
|
||||
struct psshfs_node *psn_targ;
|
||||
psn_targ = pdir->entry->pn_data;
|
||||
|
||||
puffs_setvattr(&pdir->entry->pn_va, &va);
|
||||
psn_targ->attrread = time(NULL);
|
||||
} else {
|
||||
puffs_setvattr(&pdir->va, &va);
|
||||
pdir->attrread = time(NULL);
|
||||
}
|
||||
|
||||
out:
|
||||
free(rda);
|
||||
psbuf_destroy(pb);
|
||||
}
|
||||
|
||||
static void
|
||||
readdir_getattr(struct psshfs_ctx *pctx, struct psshfs_node *psn,
|
||||
const char *basepath, int idx)
|
||||
{
|
||||
char path[MAXPATHLEN+1];
|
||||
struct psshfs_dir *pdir = psn->dir;
|
||||
struct psbuf *pb;
|
||||
struct readdirattr *rda;
|
||||
const char *entryname = pdir[idx].entryname;
|
||||
uint32_t reqid = NEXTREQ(pctx);
|
||||
|
||||
rda = emalloc(sizeof(struct readdirattr));
|
||||
rda->psn = psn;
|
||||
rda->idx = idx;
|
||||
strlcpy(rda->entryname, entryname, sizeof(rda->entryname));
|
||||
|
||||
strcpy(path, basepath);
|
||||
strcat(path, "/");
|
||||
strlcat(path, entryname, sizeof(path));
|
||||
|
||||
pb = psbuf_make(PSB_OUT);
|
||||
psbuf_req_str(pb, SSH_FXP_LSTAT, reqid, path);
|
||||
pssh_outbuf_enqueue_nocc(pctx, pb, readdir_getattr_resp, rda, reqid);
|
||||
}
|
||||
#endif
|
||||
|
||||
int
|
||||
sftp_readdir(struct puffs_cc *pcc, struct psshfs_ctx *pctx,
|
||||
struct puffs_node *pn)
|
||||
|
|
@ -122,6 +186,14 @@ sftp_readdir(struct puffs_cc *pcc, struct psshfs_ctx *pctx,
|
|||
olddir = psn->dir;
|
||||
nent = psn->dentnext;
|
||||
|
||||
/*
|
||||
* note: for the "getattr in batch" to work, this must be before
|
||||
* the attribute-getting. Otherwise times for first entries in
|
||||
* large directories might expire before the directory itself and
|
||||
* result in one-by-one attribute fetching.
|
||||
*/
|
||||
psn->dentread = time(NULL);
|
||||
|
||||
psn->dentnext = 0;
|
||||
psn->denttot = 0;
|
||||
psn->dir = NULL;
|
||||
|
|
@ -170,12 +242,19 @@ sftp_readdir(struct puffs_cc *pcc, struct psshfs_ctx *pctx,
|
|||
testd = lookup(olddir, nent, psn->dir[idx].entryname);
|
||||
if (testd) {
|
||||
psn->dir[idx].entry = testd->entry;
|
||||
psn->dir[idx].va.va_fileid
|
||||
= testd->va.va_fileid;
|
||||
psn->dir[idx].va = testd->va;
|
||||
} else {
|
||||
psn->dir[idx].entry = NULL;
|
||||
psn->dir[idx].va.va_fileid = pctx->nextino++;
|
||||
}
|
||||
#ifdef SUPERREADDIR
|
||||
/*
|
||||
* XXX: there's a dangling pointer race here if
|
||||
* the server responds to our queries out-of-order.
|
||||
* fixxxme some day
|
||||
*/
|
||||
readdir_getattr(pctx, psn, PNPATH(pn), idx);
|
||||
#endif
|
||||
psn->dir[idx].valid = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -183,7 +262,6 @@ sftp_readdir(struct puffs_cc *pcc, struct psshfs_ctx *pctx,
|
|||
out:
|
||||
/* XXX: rv */
|
||||
psn->dentnext = idx;
|
||||
psn->dentread = time(NULL);
|
||||
freedircache(olddir, nent);
|
||||
|
||||
reqid = NEXTREQ(pctx);
|
||||
|
|
@ -218,6 +296,7 @@ makenode(struct puffs_usermount *pu, struct puffs_node *parent,
|
|||
return NULL;
|
||||
}
|
||||
puffs_setvattr(&pn->pn_va, &pd->va);
|
||||
psn->attrread = pd->attrread;
|
||||
puffs_setvattr(&pn->pn_va, vap);
|
||||
|
||||
pd->entry = pn;
|
||||
|
|
@ -267,6 +346,7 @@ direnter(struct puffs_node *parent, const char *entryname)
|
|||
pd = &psn_parent->dir[i];
|
||||
pd->entryname = estrdup(entryname);
|
||||
pd->valid = 1;
|
||||
pd->attrread = 0;
|
||||
puffs_vattr_null(&pd->va);
|
||||
psn_parent->dentnext++;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue