diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index efc25927d4..f325868149 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -906,9 +906,9 @@ static void zipfileInflate( ** case. */ static int zipfileDeflate( - ZipfileTab *pTab, /* Set error message here */ const u8 *aIn, int nIn, /* Input */ - u8 **ppOut, int *pnOut /* Output */ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ ){ int nAlloc = (int)compressBound(nIn); u8 *aOut; @@ -934,7 +934,7 @@ static int zipfileDeflate( *pnOut = (int)str.total_out; }else{ sqlite3_free(aOut); - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); + *pzErr = sqlite3_mprintf("zipfile: deflate() error"); rc = SQLITE_ERROR; } deflateEnd(&str); @@ -1315,15 +1315,15 @@ static int zipfileAppendEntry( } static int zipfileGetMode( - ZipfileTab *pTab, sqlite3_value *pVal, - u32 defaultMode, /* Value to use if pVal IS NULL */ - u32 *pMode + int bIsDir, /* If true, default to directory */ + u32 *pMode, /* OUT: Mode value */ + char **pzErr /* OUT: Error message */ ){ const char *z = (const char*)sqlite3_value_text(pVal); u32 mode = 0; if( z==0 ){ - mode = defaultMode; + mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); }else if( z[0]>='0' && z[0]<='9' ){ mode = (unsigned int)sqlite3_value_int(pVal); }else{ @@ -1343,11 +1343,16 @@ static int zipfileGetMode( else if( z[i]!='-' ) goto parse_error; } } + if( (bIsDir == ((mode & S_IFDIR)==0)) ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ + return SQLITE_CONSTRAINT; + } *pMode = mode; return SQLITE_OK; parse_error: - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); return SQLITE_ERROR; } @@ -1432,7 +1437,7 @@ static int zipfileUpdate( }else{ if( bAuto || iMethod ){ int nCmp; - rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp); + rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ if( iMethod || nCmpbase.zErrMsg); } if( rc==SQLITE_OK ){ @@ -1819,6 +1817,251 @@ static int zipfileFindFunction( return 0; } +typedef struct ZipfileBuffer ZipfileBuffer; +struct ZipfileBuffer { + u8 *a; /* Pointer to buffer */ + int n; /* Size of buffer in bytes */ + int nAlloc; /* Byte allocated at a[] */ +}; + +typedef struct ZipfileCtx ZipfileCtx; +struct ZipfileCtx { + int nEntry; + ZipfileBuffer body; + ZipfileBuffer cds; +}; + +static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ + if( pBuf->n+nByte>pBuf->nAlloc ){ + u8 *aNew; + int nNew = pBuf->n ? pBuf->n*2 : 512; + int nReq = pBuf->n + nByte; + + while( nNewa, nNew); + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->a = aNew; + pBuf->nAlloc = nNew; + } + return SQLITE_OK; +} + +/* +** xStep() callback for the zipfile() aggregate. This can be called in +** any of the following ways: +** +** SELECT zipfile(name,data) ... +** SELECT zipfile(name,mode,mtime,data) ... +** SELECT zipfile(name,mode,mtime,data,method) ... +*/ +void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ + ZipfileCtx *p; /* Aggregate function context */ + ZipfileEntry e; /* New entry to add to zip archive */ + + sqlite3_value *pName = 0; + sqlite3_value *pMode = 0; + sqlite3_value *pMtime = 0; + sqlite3_value *pData = 0; + sqlite3_value *pMethod = 0; + + int bIsDir = 0; + u32 mode; + int rc = SQLITE_OK; + char *zErr = 0; + + int iMethod = -1; /* Compression method to use (0 or 8) */ + + const u8 *aData = 0; /* Possibly compressed data for new entry */ + int nData = 0; /* Size of aData[] in bytes */ + int szUncompressed = 0; /* Size of data before compression */ + u8 *aFree = 0; /* Free this before returning */ + u32 iCrc32 = 0; /* crc32 of uncompressed data */ + + char *zName = 0; /* Path (name) of new entry */ + int nName = 0; /* Size of zName in bytes */ + char *zFree = 0; /* Free this before returning */ + int nByte; + + memset(&e, 0, sizeof(e)); + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + + /* Martial the arguments into stack variables */ + if( nVal!=2 && nVal!=4 && nVal!=5 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + pName = apVal[0]; + if( nVal==2 ){ + pData = apVal[1]; + }else{ + pMode = apVal[1]; + pMtime = apVal[2]; + pData = apVal[3]; + if( nVal==5 ){ + pMethod = apVal[4]; + } + } + + /* Check that the 'name' parameter looks ok. */ + zName = (char*)sqlite3_value_text(pName); + nName = sqlite3_value_bytes(pName); + if( zName==0 ){ + zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + + /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use + ** deflate compression) or NULL (choose automatically). */ + if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ + iMethod = sqlite3_value_int64(pMethod); + if( iMethod!=0 && iMethod!=8 ){ + zErr = sqlite3_mprintf("illegal method value: %d", iMethod); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + } + + /* Now inspect the data. If this is NULL, then the new entry must be a + ** directory. Otherwise, figure out whether or not the data should + ** be deflated or simply stored in the zip archive. */ + if( sqlite3_value_type(pData)==SQLITE_NULL ){ + bIsDir = 1; + iMethod = 0; + }else{ + aData = sqlite3_value_blob(pData); + szUncompressed = nData = sqlite3_value_bytes(pData); + iCrc32 = crc32(0, aData, nData); + if( iMethod<0 || iMethod==8 ){ + int nOut = 0; + rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); + if( rc!=SQLITE_OK ){ + goto zipfile_step_out; + } + if( iMethod==8 || nOut1 && zName[nName-2]=='/' ) nName--; + } + } + + /* Assemble the ZipfileEntry object for the new zip archive entry */ + e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; + e.cds.iCompression = iMethod; + zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); + e.cds.crc32 = iCrc32; + e.cds.szCompressed = nData; + e.cds.szUncompressed = szUncompressed; + e.cds.iExternalAttr = (mode<<16); + e.cds.iOffset = p->body.n; + e.cds.nFile = nName; + e.cds.zFile = zName; + + /* Append the LFH to the body of the new archive */ + nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; + p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); + + /* Append the data to the body of the new archive */ + if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; + memcpy(&p->body.a[p->body.n], aData, nData); + p->body.n += nData; + + /* Append the CDS record to the directory of the new archive */ + nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; + p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + + /* Increment the count of entries in the archive */ + p->nEntry++; + + zipfile_step_out: + sqlite3_free(aFree); + sqlite3_free(zFree); + if( rc ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free(zErr); +} + +/* +** xFinalize() callback for zipfile aggregate function. +*/ +void zipfileFinal(sqlite3_context *pCtx){ + ZipfileCtx *p; + ZipfileEOCD eocd; + int nZip; + u8 *aZip; + + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 || p->nEntry==0 ) return; + + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = p->nEntry; + eocd.nEntryTotal = p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; + + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree); + } + + sqlite3_free(p->body.a); + sqlite3_free(p->cds.a); +} + + /* ** Register the "zipfile" virtual table. */ @@ -1849,6 +2092,11 @@ static int zipfileRegister(sqlite3 *db){ int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, + zipfileStep, zipfileFinal + ); + } return rc; } #else /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/manifest b/manifest index 7a5772a6de..09731fddf8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE). -D 2018-01-29T16:22:39.280 +C Add\saggregate\sfunction\szipfile()\sto\sthe\szipfile\sextension.\sFor\scomposing\snew\nzip\sarchives\sin\smemory. +D 2018-01-29T18:41:07.175 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 398f53469ed90076dd5d69494eb7f1bd01a42d402fc5dbe92374178887244805 +F ext/misc/zipfile.c 73d932caf69fea8ba42b58ce2cea0fbea2c126a355498fd215008f61651ec661 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 1b213bdc31eddd4a41042875dbdb29b6ea12b7da7a372a8eb07f61d0d76d800f +F test/zipfile.test 12199631759785b0185ceb1ff3cffdcc91985ef61475ab1c69139686b09a87c8 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 -R 3ff6b96757835180119359cd3166f741 +P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 +R 5f1a0bf3bc3038aabd0aee2df4b1bd63 U dan -Z 6c5f0f4d34834e8c6ccf2da8ff3a018a +Z eba1cb16b20a1e223d496488c50326dc diff --git a/manifest.uuid b/manifest.uuid index 9eae811f9a..78aea7435e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 \ No newline at end of file +e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index e74f81ed1d..04a475244b 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -22,33 +22,43 @@ if {[catch {load_static_extension db zipfile} error]} { finish_test; return } -proc do_zipfile_blob_test {tn file} { - set res1 [ - db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($file) } - ] - - set fd [open $file] +proc readfile {f} { + set fd [open $f] fconfigure $fd -translation binary -encoding binary set data [read $fd] close $fd + set data +} - set res2 [db eval { - SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) - }] +# Argument $file is the name of a zip archive on disk. This function +# executes test cases to check that the results of each of the following +# are the same: +# +# SELECT * FROM zipfile($file) +# SELECT * FROM zipfile( readfile($file) ) +# SELECT * FROM zipfile( +# (SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file)) +# ) +# +proc do_zipfile_blob_test {tn file} { - uplevel [list do_test $tn.1 [list set {} $res2] $res1] + db func r readfile + set q1 {SELECT name,mode,mtime,method,quote(data) FROM zipfile($file)} + set q2 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( r($file) )} + set q3 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( + ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) + )} - set T "$file.test_zip" - set fd [open $T w] - fconfigure $fd -translation binary -encoding binary - puts -nonewline $fd $data - close $fd - set res3 [ - db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($T) } - ] + set r1 [db eval $q1] + set r2 [db eval $q2] + set r3 [db eval $q3] + #puts $r1 + #puts $r2 + #puts $r3 - uplevel [list do_test $tn.2 [list set {} $res3] $res1] + uplevel [list do_test $tn.1 [list set {} $r2] $r1] + uplevel [list do_test $tn.1 [list set {} $r3] $r1] } forcedelete test.zip