Have zipfile support DELETE commands.

FossilOrigin-Name: 01d4e866fb7b01aeada537d41c4a47747c7810e2028f51077ee5b8b78c348954
This commit is contained in:
dan 2017-12-30 18:32:27 +00:00
parent db0cb303ad
commit 0cde0c62b1
4 changed files with 202 additions and 114 deletions

View File

@ -201,40 +201,45 @@ struct ZipfileLFH {
u16 nExtra;
};
/*
** Cursor type for recursively iterating through a directory structure.
*/
typedef struct ZipfileCsr ZipfileCsr;
struct ZipfileCsr {
sqlite3_vtab_cursor base; /* Base class - must be first */
i64 iRowid; /* Rowid for current row */
FILE *pFile; /* Zip file */
i64 nByte; /* Size of zip file on disk */
int bEof; /* True when at EOF */
i64 iNextOff; /* Offset of next record in central directory */
ZipfileEOCD eocd; /* Parse of central directory record */
ZipfileCDS cds; /* Central Directory Structure */
ZipfileLFH lfh; /* Local File Header for current entry */
i64 iDataOff; /* Offset in zipfile to data */
u32 mTime; /* Extended mtime value */
int flags; /* Flags byte (see below for bits) */
};
/*
** Values for ZipfileCsr.flags.
*/
#define ZIPFILE_MTIME_VALID 0x0001
typedef struct ZipfileEntry ZipfileEntry;
struct ZipfileEntry {
char *zPath; /* Path of zipfile entry */
i64 iRowid; /* Rowid for this value if queried */
u8 *aCdsEntry; /* Buffer containing entire CDS entry */
int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */
int bDeleted; /* True if entry has been deleted */
ZipfileEntry *pNext; /* Next element in in-memory CDS */
};
/*
** Cursor type for recursively iterating through a directory structure.
*/
typedef struct ZipfileCsr ZipfileCsr;
struct ZipfileCsr {
sqlite3_vtab_cursor base; /* Base class - must be first */
int bEof; /* True when at EOF */
/* Used outside of write transactions */
FILE *pFile; /* Zip file */
i64 iNextOff; /* Offset of next record in central directory */
ZipfileEOCD eocd; /* Parse of central directory record */
/* Used inside write transactions */
ZipfileEntry *pCurrent;
ZipfileCDS cds; /* Central Directory Structure */
ZipfileLFH lfh; /* Local File Header for current entry */
i64 iDataOff; /* Offset in zipfile to data */
u32 mTime; /* Extended mtime value */
int flags; /* Flags byte (see below for bits) */
ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */
};
/*
** Values for ZipfileCsr.flags.
*/
#define ZIPFILE_MTIME_VALID 0x0001
typedef struct ZipfileTab ZipfileTab;
struct ZipfileTab {
sqlite3_vtab base; /* Base class - must be first */
@ -242,6 +247,7 @@ struct ZipfileTab {
u8 *aBuffer; /* Temporary buffer used for various tasks */
/* The following are used by write transactions only */
ZipfileCsr *pCsrList; /* List of cursors */
ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */
FILE *pWriteFd; /* File handle open on zip archive */
@ -335,9 +341,16 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
** by zipfileOpen().
*/
static void zipfileResetCursor(ZipfileCsr *pCsr){
ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab);
ZipfileCsr **pp;
/* Remove this cursor from the ZipfileTab.pCsrList list. */
for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){
if( *pp==pCsr ) *pp = pCsr->pCsrNext;
}
sqlite3_free(pCsr->cds.zFile);
pCsr->cds.zFile = 0;
pCsr->iRowid = 0;
pCsr->bEof = 0;
if( pCsr->pFile ){
fclose(pCsr->pFile);
@ -438,18 +451,25 @@ static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
static int zipfileReadCDS(ZipfileCsr *pCsr){
char **pzErr = &pCsr->base.pVtab->zErrMsg;
u8 *aRead = zipfileCsrBuffer(pCsr);
int rc;
u8 *aRead;
int rc = SQLITE_OK;
sqlite3_free(pCsr->cds.zFile);
pCsr->cds.zFile = 0;
rc = zipfileReadData(
pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
);
if( pCsr->pCurrent==0 ){
aRead = zipfileCsrBuffer(pCsr);
rc = zipfileReadData(
pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
);
}else{
aRead = pCsr->pCurrent->aCdsEntry;
}
if( rc==SQLITE_OK ){
u32 sig = zipfileRead32(aRead);
if( sig!=ZIPFILE_SIGNATURE_CDS ){
assert( pCsr->pCurrent==0 );
zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
rc = SQLITE_ERROR;
}else{
@ -463,7 +483,9 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
pCsr->cds.crc32 = zipfileRead32(aRead);
pCsr->cds.szCompressed = zipfileRead32(aRead);
pCsr->cds.szUncompressed = zipfileRead32(aRead);
assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF );
assert( pCsr->pCurrent
|| aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF
);
pCsr->cds.nFile = zipfileRead16(aRead);
pCsr->cds.nExtra = zipfileRead16(aRead);
pCsr->cds.nComment = zipfileRead16(aRead);
@ -471,13 +493,16 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
pCsr->cds.iInternalAttr = zipfileRead16(aRead);
pCsr->cds.iExternalAttr = zipfileRead32(aRead);
pCsr->cds.iOffset = zipfileRead32(aRead);
assert( pCsr->pCurrent
|| aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ
);
assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ );
nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
aRead = zipfileCsrBuffer(pCsr);
pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
if( pCsr->pCurrent==0 ){
nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
aRead = zipfileCsrBuffer(pCsr);
pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
}
if( rc==SQLITE_OK ){
pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
@ -522,13 +547,19 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
return rc;
}
static FILE *zipfileGetFd(ZipfileCsr *pCsr){
if( pCsr->pFile ) return pCsr->pFile;
return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
}
static int zipfileReadLFH(ZipfileCsr *pCsr){
FILE *pFile = zipfileGetFd(pCsr);
char **pzErr = &pCsr->base.pVtab->zErrMsg;
static const int szFix = ZIPFILE_LFH_FIXED_SZ;
u8 *aRead = zipfileCsrBuffer(pCsr);
int rc;
rc = zipfileReadData(pCsr->pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
rc = zipfileReadData(pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
if( rc==SQLITE_OK ){
u32 sig = zipfileRead32(aRead);
if( sig!=ZIPFILE_SIGNATURE_LFH ){
@ -561,19 +592,31 @@ static int zipfileReadLFH(ZipfileCsr *pCsr){
*/
static int zipfileNext(sqlite3_vtab_cursor *cur){
ZipfileCsr *pCsr = (ZipfileCsr*)cur;
i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
int rc = SQLITE_OK;
pCsr->flags = 0;
if( pCsr->iNextOff>=iEof ){
pCsr->bEof = 1;
if( pCsr->pCurrent==0 ){
i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
if( pCsr->iNextOff>=iEof ){
pCsr->bEof = 1;
}
}else{
pCsr->iRowid++;
pCsr->flags = 0;
assert( pCsr->pFile==0 );
do {
pCsr->pCurrent = pCsr->pCurrent->pNext;
}while( pCsr->pCurrent && pCsr->pCurrent->bDeleted );
if( pCsr->pCurrent==0 ){
pCsr->bEof = 1;
}
}
if( pCsr->bEof==0 ){
rc = zipfileReadCDS(pCsr);
if( rc==SQLITE_OK ){
rc = zipfileReadLFH(pCsr);
}
}
return rc;
}
@ -658,7 +701,8 @@ static int zipfileColumn(
if( aBuf==0 ){
rc = SQLITE_NOMEM;
}else{
rc = zipfileReadData(pCsr->pFile, aBuf, sz, pCsr->iDataOff,
FILE *pFile = zipfileGetFd(pCsr);
rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
&pCsr->base.pVtab->zErrMsg
);
}
@ -678,13 +722,15 @@ static int zipfileColumn(
}
/*
** Return the rowid for the current row. In this implementation, the
** first row returned is assigned rowid value 1, and each subsequent
** row a value 1 more than that of the previous.
** Return the rowid for the current row.
*/
static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
ZipfileCsr *pCsr = (ZipfileCsr*)cur;
*pRowid = pCsr->iRowid;
if( pCsr->pCurrent ){
*pRowid = pCsr->pCurrent->iRowid;
}else{
*pRowid = pCsr->cds.iOffset;
}
return SQLITE_OK;
}
@ -771,32 +817,39 @@ static int zipfileFilter(
zipfileResetCursor(pCsr);
assert( idxNum==argc && (idxNum==0 || idxNum==1) );
if( idxNum==0 ){
ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
if( pTab->zFile ){
zFile = pTab->zFile;
if( zFile==0 ){
/* Error. This is an eponymous virtual table and the user has not
** supplied a file name. */
zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
return SQLITE_ERROR;
}
}else if( idxNum==0 ){
/* Error. This is an eponymous virtual table and the user has not
** supplied a file name. */
zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
return SQLITE_ERROR;
}else{
zFile = (const char*)sqlite3_value_text(argv[0]);
}
pCsr->pFile = fopen(zFile, "rb");
if( pCsr->pFile==0 ){
zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
rc = SQLITE_ERROR;
}else{
rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
if( rc==SQLITE_OK ){
pCsr->iNextOff = pCsr->eocd.iOffset;
rc = zipfileNext(cur);
}else if( rc==SQLITE_EMPTY ){
rc = SQLITE_OK;
pCsr->bEof = 1;
if( pTab->pWriteFd==0 ){
pCsr->pFile = fopen(zFile, "rb");
if( pCsr->pFile==0 ){
zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
rc = SQLITE_ERROR;
}else{
rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
if( rc==SQLITE_OK ){
pCsr->iNextOff = pCsr->eocd.iOffset;
rc = zipfileNext(cur);
}else if( rc==SQLITE_EMPTY ){
rc = SQLITE_OK;
pCsr->bEof = 1;
}
}
}else{
ZipfileEntry e;
memset(&e, 0, sizeof(e));
e.pNext = pTab->pFirstEntry;
pCsr->pCurrent = &e;
rc = zipfileNext(cur);
assert( pCsr->pCurrent!=&e );
}
return rc;
@ -840,9 +893,11 @@ static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){
assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
assert( pNew->pNext==0 );
if( pTab->pFirstEntry==0 ){
pNew->iRowid = 1;
pTab->pFirstEntry = pTab->pLastEntry = pNew;
}else{
assert( pTab->pLastEntry->pNext==0 );
pNew->iRowid = pTab->pLastEntry->iRowid+1;
pTab->pLastEntry->pNext = pNew;
pTab->pLastEntry = pNew;
}
@ -884,12 +939,12 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
memset(pNew, 0, sizeof(ZipfileEntry));
pNew->zPath = (char*)&pNew[1];
memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
pNew->zPath[nFile] = '\0';
pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
pNew->pNext = 0;
memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
zipfileAddEntry(pTab, pNew);
}
@ -922,10 +977,10 @@ static ZipfileEntry *zipfileNewEntry(
);
if( pNew ){
memset(pNew, 0, sizeof(ZipfileEntry));
pNew->zPath = (char*)&pNew[1];
pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
pNew->pNext = 0;
memcpy(pNew->zPath, zPath, nPath+1);
aWrite = pNew->aCdsEntry;
@ -1057,38 +1112,19 @@ static int zipfileUpdate(
u8 *pFree = 0; /* Free this */
ZipfileCDS cds; /* New Central Directory Structure entry */
assert( pTab->zFile );
assert( pTab->pWriteFd );
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
pTab->base.zErrMsg = sqlite3_mprintf(
"zipfile: UPDATE/DELETE not yet supported"
);
return SQLITE_ERROR;
}
/* This table is only writable if a default archive path was specified
** as part of the CREATE VIRTUAL TABLE statement. */
if( pTab->zFile==0 ){
pTab->base.zErrMsg = sqlite3_mprintf(
"zipfile: writing requires a default archive"
);
return SQLITE_ERROR;
}
/* Open a write fd on the file. Also load the entire central directory
** structure into memory. During the transaction any new file data is
** appended to the archive file, but the central directory is accumulated
** in main-memory until the transaction is committed. */
if( pTab->pWriteFd==0 ){
pTab->pWriteFd = fopen(pTab->zFile, "ab+");
if( pTab->pWriteFd==0 ){
pTab->base.zErrMsg = sqlite3_mprintf(
"zipfile: failed to open file %s for writing", pTab->zFile
);
return SQLITE_ERROR;
i64 iDelete = sqlite3_value_int64(apVal[0]);
ZipfileEntry *p;
for(p=pTab->pFirstEntry; p; p=p->pNext){
if( p->iRowid==iDelete ){
p->bDeleted = 1;
break;
}
}
fseek(pTab->pWriteFd, 0, SEEK_END);
pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
rc = zipfileLoadDirectory(pTab);
if( rc!=SQLITE_OK ) return rc;
if( nVal==1 ) return SQLITE_OK;
}
zPath = (const char*)sqlite3_value_text(apVal[2]);
@ -1199,7 +1235,41 @@ static void zipfileCleanupTransaction(ZipfileTab *pTab){
}
static int zipfileBegin(sqlite3_vtab *pVtab){
return SQLITE_OK;
ZipfileTab *pTab = (ZipfileTab*)pVtab;
int rc = SQLITE_OK;
assert( pTab->pWriteFd==0 );
/* This table is only writable if a default archive path was specified
** as part of the CREATE VIRTUAL TABLE statement. */
if( pTab->zFile==0 ){
pTab->base.zErrMsg = sqlite3_mprintf(
"zipfile: writing requires a default archive"
);
return SQLITE_ERROR;
}
/* Open a write fd on the file. Also load the entire central directory
** structure into memory. During the transaction any new file data is
** appended to the archive file, but the central directory is accumulated
** in main-memory until the transaction is committed. */
pTab->pWriteFd = fopen(pTab->zFile, "ab+");
if( pTab->pWriteFd==0 ){
pTab->base.zErrMsg = sqlite3_mprintf(
"zipfile: failed to open file %s for writing", pTab->zFile
);
rc = SQLITE_ERROR;
}else{
fseek(pTab->pWriteFd, 0, SEEK_END);
pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
rc = zipfileLoadDirectory(pTab);
}
if( rc!=SQLITE_OK ){
zipfileCleanupTransaction(pTab);
}
return rc;
}
static int zipfileCommit(sqlite3_vtab *pVtab){
@ -1211,8 +1281,9 @@ static int zipfileCommit(sqlite3_vtab *pVtab){
ZipfileEOCD eocd;
int nEntry = 0;
/* Write out all entries */
/* Write out all undeleted entries */
for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){
if( p->bDeleted ) continue;
rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry);
nEntry++;
}

View File

@ -1,5 +1,5 @@
C Rearrange\sthings\sa\sbit\sso\sthat\swriting\sto\sa\szipfile\sdoes\snot\sinvert\sthe\sorder\nof\sobjects\sit\scontains.
D 2017-12-30T14:26:29.195
C Have\szipfile\ssupport\sDELETE\scommands.
D 2017-12-30T18:32:27.545
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
F ext/misc/zipfile.c cf093f797331e51fa6c0358698bc89d08a88d06cdf11484ee4ea9d079145289b
F ext/misc/zipfile.c 2df8f94003903fe3cc104b807418c54e68040964d4319c522ac2f485152f5abd
F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@ -1596,7 +1596,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
F test/zipfile.test 63059e59024cacd01ba4c32a445cf75aac21393ff4460bddd1dc1ef4b78dd8bc
F test/zipfile.test d4f342ca918fd4d7981a12c1523f5041074cc592f25fecce4ee11446cc984f56
F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@ -1693,7 +1693,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d
R 64b99b9622049272ef5b18e73b616095
P f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
R 145246df93a24ee9820cce32c6b5afb4
U dan
Z 6ef5282c9c40bbaf3432c6295db23abc
Z 640cac8c8ac89e53412f930b0b057c7a

View File

@ -1 +1 @@
f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
01d4e866fb7b01aeada537d41c4a47747c7810e2028f51077ee5b8b78c348954

View File

@ -29,13 +29,15 @@ do_execsql_test 1.0 {
5 method {} 0 {} 0
}
do_execsql_test 1.1 {
do_execsql_test 1.1.1 {
INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
}
do_execsql_test 1.1.2 {
INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
}
do_execsql_test 1.2 {
SELECT name, mtime, data FROM zipfile('test.zip');
SELECT name, mtime, data FROM zipfile('test.zip')
} {
f.txt 1000000000 abcde
g.txt 1000000002 12345
@ -56,5 +58,20 @@ do_execsql_test 1.4 {
h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
}
do_execsql_test 1.5.1 {
BEGIN;
INSERT INTO zz VALUES('i.txt', '-rw-r--r--', 1000000006, 5, 'zxcvb', 0);
SELECT name FROM zz;
COMMIT;
} {f.txt g.txt h.txt i.txt}
do_execsql_test 1.5.2 {
SELECT name FROM zz;
} {f.txt g.txt h.txt i.txt}
do_execsql_test 1.6.0 {
DELETE FROM zz WHERE name='g.txt';
SELECT name FROM zz;
} {f.txt h.txt i.txt}
finish_test