From 1ad3f61b810e48ad8d6f0b59152689fc10bdfb28 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Dec 2017 20:22:02 +0000 Subject: [PATCH] Enhance virtual table "fsdir" in ext/misc/fileio.c. Add support for "-C" to the shell command's ".ar c" command. FossilOrigin-Name: 0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef --- ext/misc/fileio.c | 230 +++++++++++++++++++++++++++------------------- manifest | 16 ++-- manifest.uuid | 2 +- src/shell.c.in | 11 +-- test/shell8.test | 10 ++ 5 files changed, 158 insertions(+), 111 deletions(-) diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index db69357008..d3c00b4b1f 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -42,7 +42,7 @@ SQLITE_EXTENSION_INIT1 #include -#define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)" +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" static void readFileContents(sqlite3_context *ctx, const char *zName){ FILE *in; @@ -166,29 +166,36 @@ static void writefileFunc( #ifndef SQLITE_OMIT_VIRTUALTABLE /* +** Cursor type for recursively iterating through a directory structure. */ typedef struct fsdir_cursor fsdir_cursor; +typedef struct FsdirLevel FsdirLevel; + +struct FsdirLevel { + DIR *pDir; /* From opendir() */ + char *zDir; /* Name of directory (nul-terminated) */ +}; + struct fsdir_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - int eType; /* One of FSDIR_DIR or FSDIR_ENTRY */ - DIR *pDir; /* From opendir() */ + + int nLvl; /* Number of entries in aLvl[] array */ + int iLvl; /* Index of current entry */ + FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ + + const char *zBase; + int nBase; + struct stat sStat; /* Current lstat() results */ - char *zDir; /* Directory to read */ - int nDir; /* Value of strlen(zDir) */ char *zPath; /* Path to current entry */ - int bEof; sqlite3_int64 iRowid; /* Current rowid */ }; typedef struct fsdir_tab fsdir_tab; struct fsdir_tab { sqlite3_vtab base; /* Base class - must be first */ - int eType; /* One of FSDIR_DIR or FSDIR_ENTRY */ }; -#define FSDIR_DIR 0 -#define FSDIR_ENTRY 1 - /* ** Construct a new fsdir virtual table object. */ @@ -202,12 +209,11 @@ static int fsdirConnect( fsdir_tab *pNew = 0; int rc; - rc = sqlite3_declare_vtab(db, FSDIR_SCHEMA); + rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); if( rc==SQLITE_OK ){ pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); - pNew->eType = (pAux==0 ? FSDIR_DIR : FSDIR_ENTRY); } *ppVtab = (sqlite3_vtab*)pNew; return rc; @@ -229,56 +235,107 @@ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ pCur = sqlite3_malloc( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); - pCur->eType = ((fsdir_tab*)p)->eType; + pCur->iLvl = -1; *ppCursor = &pCur->base; return SQLITE_OK; } +static void fsdirResetCursor(fsdir_cursor *pCur){ + int i; + for(i=0; i<=pCur->iLvl; i++){ + FsdirLevel *pLvl = &pCur->aLvl[i]; + if( pLvl->pDir ) closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + } + sqlite3_free(pCur->zPath); + pCur->aLvl = 0; + pCur->zPath = 0; + pCur->zBase = 0; + pCur->nBase = 0; + pCur->iLvl = -1; + pCur->iRowid = 1; +} + /* ** Destructor for an fsdir_cursor. */ static int fsdirClose(sqlite3_vtab_cursor *cur){ fsdir_cursor *pCur = (fsdir_cursor*)cur; - if( pCur->pDir ) closedir(pCur->pDir); - sqlite3_free(pCur->zDir); - sqlite3_free(pCur->zPath); + + fsdirResetCursor(pCur); + sqlite3_free(pCur->aLvl); sqlite3_free(pCur); return SQLITE_OK; } +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + + /* ** Advance an fsdir_cursor to its next row of output. */ static int fsdirNext(sqlite3_vtab_cursor *cur){ fsdir_cursor *pCur = (fsdir_cursor*)cur; - struct dirent *pEntry; + mode_t m = pCur->sStat.st_mode; - if( pCur->eType==FSDIR_ENTRY ){ - pCur->bEof = 1; - return SQLITE_OK; - } - - sqlite3_free(pCur->zPath); - pCur->zPath = 0; - - while( 1 ){ - pEntry = readdir(pCur->pDir); - if( pEntry ){ - if( strcmp(pEntry->d_name, ".") - && strcmp(pEntry->d_name, "..") - ){ - pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zDir, pEntry->d_name); - if( pCur->zPath==0 ) return SQLITE_NOMEM; - lstat(pCur->zPath, &pCur->sStat); - break; - } - }else{ - pCur->bEof = 1; - break; + pCur->iRowid++; + if( S_ISDIR(m) ){ + /* Descend into this directory */ + int iNew = pCur->iLvl + 1; + FsdirLevel *pLvl; + if( iNew>=pCur->nLvl ){ + int nNew = iNew+1; + int nByte = nNew*sizeof(FsdirLevel); + FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl)); + pCur->aLvl = aNew; + pCur->nLvl = nNew; + } + pCur->iLvl = iNew; + pLvl = &pCur->aLvl[iNew]; + + pLvl->zDir = pCur->zPath; + pCur->zPath = 0; + pLvl->pDir = opendir(pLvl->zDir); + if( pLvl->pDir==0 ){ + fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + return SQLITE_ERROR; } } - pCur->iRowid++; + while( pCur->iLvl>=0 ){ + FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl]; + struct dirent *pEntry = readdir(pLvl->pDir); + if( pEntry ){ + if( pEntry->d_name[0]=='.' ){ + if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue; + if( pEntry->d_name[1]=='\0' ) continue; + } + sqlite3_free(pCur->zPath); + pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + if( lstat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + pLvl->pDir = 0; + pLvl->zDir = 0; + pCur->iLvl--; + } + + /* EOF */ + sqlite3_free(pCur->zPath); + pCur->zPath = 0; return SQLITE_OK; } @@ -294,13 +351,7 @@ static int fsdirColumn( fsdir_cursor *pCur = (fsdir_cursor*)cur; switch( i ){ case 0: { /* name */ - const char *zName; - if( pCur->eType==FSDIR_DIR ){ - zName = &pCur->zPath[pCur->nDir+1]; - }else{ - zName = pCur->zPath; - } - sqlite3_result_text(ctx, zName, -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT); break; } @@ -308,11 +359,11 @@ static int fsdirColumn( sqlite3_result_int64(ctx, pCur->sStat.st_mode); break; - case 2: /* mode */ + case 2: /* mtime */ sqlite3_result_int64(ctx, pCur->sStat.st_mtime); break; - case 3: { + case 3: { /* data */ mode_t m = pCur->sStat.st_mode; if( S_ISDIR(m) ){ sqlite3_result_null(ctx); @@ -361,14 +412,7 @@ static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int fsdirEof(sqlite3_vtab_cursor *cur){ fsdir_cursor *pCur = (fsdir_cursor*)cur; - return pCur->bEof; -} - -static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); - va_end(ap); + return (pCur->zPath==0); } /* @@ -382,51 +426,38 @@ static int fsdirFilter( const char *zDir = 0; fsdir_cursor *pCur = (fsdir_cursor*)cur; - sqlite3_free(pCur->zDir); - pCur->iRowid = 0; - pCur->zDir = 0; - pCur->bEof = 0; - if( pCur->pDir ){ - closedir(pCur->pDir); - pCur->pDir = 0; - } + fsdirResetCursor(pCur); if( idxNum==0 ){ fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); return SQLITE_ERROR; } - assert( argc==1 ); + assert( argc==idxNum && (argc==1 || argc==2) ); zDir = (const char*)sqlite3_value_text(argv[0]); if( zDir==0 ){ fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); return SQLITE_ERROR; } + if( argc==2 ){ + pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + } + if( pCur->zBase ){ + pCur->nBase = strlen(pCur->zBase)+1; + pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir); + }else{ + pCur->zPath = sqlite3_mprintf("%s", zDir); + } - pCur->zDir = sqlite3_mprintf("%s", zDir); - if( pCur->zDir==0 ){ + if( pCur->zPath==0 ){ return SQLITE_NOMEM; } - - if( pCur->eType==FSDIR_ENTRY ){ - int rc = lstat(pCur->zDir, &pCur->sStat); - if( rc ){ - fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zDir); - }else{ - pCur->zPath = sqlite3_mprintf("%s", pCur->zDir); - if( pCur->zPath==0 ) return SQLITE_NOMEM; - } - return SQLITE_OK; - }else{ - pCur->nDir = strlen(pCur->zDir); - pCur->pDir = opendir(zDir); - if( pCur->pDir==0 ){ - fsdirSetErrmsg(pCur, "error in opendir(\"%s\")", zDir); - return SQLITE_ERROR; - } - - return fsdirNext(cur); + if( lstat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; } + + return SQLITE_OK; } /* @@ -450,24 +481,33 @@ static int fsdirBestIndex( sqlite3_index_info *pIdxInfo ){ int i; /* Loop over constraints */ + int idx4 = -1; + int idx5 = -1; const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ if( pConstraint->usable==0 ) continue; if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - if( pConstraint->iColumn!=4 ) continue; - break; + if( pConstraint->iColumn==4 ) idx4 = i; + if( pConstraint->iColumn==5 ) idx5 = i; } - if( inConstraint ){ - pIdxInfo->aConstraintUsage[i].omit = 1; - pIdxInfo->aConstraintUsage[i].argvIndex = 1; - pIdxInfo->idxNum = 1; - pIdxInfo->estimatedCost = 10.0; - }else{ + if( idx4<0 ){ pIdxInfo->idxNum = 0; pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50); + }else{ + pIdxInfo->aConstraintUsage[idx4].omit = 1; + pIdxInfo->aConstraintUsage[idx4].argvIndex = 1; + if( idx5>=0 ){ + pIdxInfo->aConstraintUsage[idx5].omit = 1; + pIdxInfo->aConstraintUsage[idx5].argvIndex = 2; + pIdxInfo->idxNum = 2; + pIdxInfo->estimatedCost = 10.0; + }else{ + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 100.0; + } } return SQLITE_OK; diff --git a/manifest b/manifest index 1916921b89..d95b1b4be9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\s-C\sto\s".ar\sx". -D 2017-12-09T18:28:22.916 +C Enhance\svirtual\stable\s"fsdir"\sin\sext/misc/fileio.c.\sAdd\ssupport\sfor\s"-C"\sto\nthe\sshell\scommand's\s".ar\sc"\scommand. +D 2017-12-11T20:22:02.045 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 @@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 -F ext/misc/fileio.c c84ec9c399657bd95914e7c4e9aefcc148cb86fe3ab7f2102e9557c861dd0d51 +F ext/misc/fileio.c 238d0c65a14bda99748285527543273e0248bc428c01c3955ec45acb8738db3b F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984 @@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in 8e57abbd26d7d2344ba752be0f86b2cbf93ad73ca28c651786392fbfd3b512ba +F src/shell.c.in 4c7a2c1226216dfad7978642c958143f0fdfee4f09aef2add1e73e3601dd7d33 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1214,7 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f -F test/shell8.test 0e8e064da50c92df8eb514202dcfd0020f71762250066bf41ed098e0ff5f0e3d +F test/shell8.test d04773f4e8cdad62d42a1b13bf4c1182109880441689a2f5d5001e190ec04831 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce @@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5 -R e8ce05071b1dec8c5193e0b4d7b226fe +P 8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515 +R ddf59f7efc51e054de53889e9f879e48 U dan -Z f3cf78b2d0eddbe511b3f9a244339aaa +Z c8a2667de38f55deb8cffcf1545f4d0c diff --git a/manifest.uuid b/manifest.uuid index b240d7b3d5..1e0e922cc2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515 \ No newline at end of file +0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 431329579e..0e32cbe1e2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4285,6 +4285,7 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){ return rc; } + /* ** Implementation of .ar "Create" command. ** @@ -4298,11 +4299,7 @@ static int arCreateCommand( ArCommand *pAr /* Command arguments and options */ ){ const char *zSql = - "WITH f(n, m, t, d) AS (" - " SELECT name, mode, mtime, data FROM fsentry(:1) UNION ALL " - " SELECT n || '/' || name, mode, mtime, data " - " FROM f, fsdir(n) WHERE (m&?)" - ") SELECT * FROM f"; + "SELECT name, mode, mtime, data FROM fsdir(?, ?)"; const char *zSqlar = "CREATE TABLE IF NOT EXISTS sqlar(" @@ -4322,17 +4319,17 @@ static int arCreateCommand( Bytef *aCompress = 0; /* Compression buffer */ int nCompress = 0; /* Size of compression buffer */ - + rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0); if( rc!=SQLITE_OK ) return rc; rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0); shellPrepare(p, &rc, zInsert, &pInsert); shellPrepare(p, &rc, zSql, &pStmt); + sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC); for(i=0; inArg && rc==SQLITE_OK; i++){ sqlite3_bind_text(pStmt, 1, pAr->azArg[i], -1, SQLITE_STATIC); - sqlite3_bind_int(pStmt, 2, S_IFDIR); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int sz; const char *zName = (const char*)sqlite3_column_text(pStmt, 0); diff --git a/test/shell8.test b/test/shell8.test index 2d2fc7b94d..b3189d4215 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -87,6 +87,16 @@ do_test 1.2 { dir_to_list ar3/ar1 } $expected +do_test 1.3 { + file delete -force ar3 + file mkdir ar3 + + forcedelete test_ar.db + catchcmd test_ar.db ".ar cC ar1 file1 file2 dir1" + catchcmd test_ar.db ".ar xC ar3" + dir_to_list ar3 +} $expected + finish_test