From 7c15ac1adac225c405d4632d38b41099997e186a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Jan 2018 19:59:59 +0000 Subject: [PATCH] Update the zipfile module so that it matches the documentation. FossilOrigin-Name: 7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f --- ext/misc/zipfile.c | 264 ++++++++++++++++++++++++++------------------- manifest | 16 +-- manifest.uuid | 2 +- src/shell.c.in | 4 +- test/zipfile.test | 21 ++-- 5 files changed, 175 insertions(+), 132 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index d2715c4400..c99bac080e 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -42,17 +42,18 @@ typedef unsigned long u32; #define MIN(a,b) ((a)<(b) ? (a) : (b)) #endif -#define ZIPFILE_SCHEMA "CREATE TABLE y(" \ - "name, /* Name of file in zip archive */" \ - "mode, /* POSIX mode for file */" \ - "mtime, /* Last modification time in seconds since epoch */" \ - "sz, /* Size of object */" \ - "data, /* Data stored in zip file (possibly compressed) */" \ - "method, /* Compression method (integer) */" \ - "f HIDDEN /* Name of zip file */" \ +#define ZIPFILE_SCHEMA "CREATE TABLE y(" \ + "name, /* 0: Name of file in zip archive */" \ + "mode, /* 1: POSIX mode for file */" \ + "mtime, /* 2: Last modification time in seconds since epoch */" \ + "sz, /* 3: Size of object */" \ + "rawdata, /* 4: Raw data */" \ + "data, /* 5: Uncompressed data */" \ + "method, /* 6: Compression method (integer) */" \ + "file HIDDEN /* Name of zip file */" \ ");" -#define ZIPFILE_F_COLUMN_IDX 6 /* Index of column "f" in the above */ +#define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "f" in the above */ #define ZIPFILE_BUFFER_SIZE (64*1024) @@ -671,6 +672,80 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){ ((res.tm_year-80) << 9)); } +static void zipfileInflate( + sqlite3_context *pCtx, /* Store error here, if any */ + const u8 *aIn, /* Compressed data */ + int nIn, /* Size of buffer aIn[] in bytes */ + int nOut /* Expected output size */ +){ + u8 *aRes = sqlite3_malloc(nOut); + if( aRes==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + int err; + z_stream str; + memset(&str, 0, sizeof(str)); + + str.next_in = (Byte*)aIn; + str.avail_in = nIn; + str.next_out = (Byte*)aRes; + str.avail_out = nOut; + + err = inflateInit2(&str, -15); + if( err!=Z_OK ){ + zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err); + }else{ + err = inflate(&str, Z_NO_FLUSH); + if( err!=Z_STREAM_END ){ + zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); + }else{ + sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT); + } + } + sqlite3_free(aRes); + inflateEnd(&str); + } +} + +static int zipfileDeflate( + ZipfileTab *pTab, /* Set error message here */ + const u8 *aIn, int nIn, /* Input */ + u8 **ppOut, int *pnOut /* Output */ +){ + int nAlloc = (int)compressBound(nIn); + u8 *aOut; + int rc; + + aOut = (u8*)sqlite3_malloc(nAlloc); + if( aOut==0 ){ + rc = SQLITE_NOMEM; + }else{ + int res; + z_stream str; + memset(&str, 0, sizeof(str)); + str.next_in = (z_const Bytef*)aIn; + str.avail_in = nIn; + str.next_out = aOut; + str.avail_out = nAlloc; + + deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + res = deflate(&str, Z_FINISH); + + if( res==Z_STREAM_END ){ + *ppOut = aOut; + *pnOut = (int)str.total_out; + }else{ + sqlite3_free(aOut); + pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); + rc = SQLITE_ERROR; + } + deflateEnd(&str); + } + + return rc; +} + + /* ** Return values of columns for the row at which the series_cursor ** is currently pointing. @@ -703,26 +778,33 @@ static int zipfileColumn( sqlite3_result_int64(ctx, pCsr->cds.szUncompressed); break; } - case 4: { /* data */ - int sz = pCsr->cds.szCompressed; - if( sz>0 ){ - u8 *aBuf = sqlite3_malloc(sz); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - FILE *pFile = zipfileGetFd(pCsr); - rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff, - &pCsr->base.pVtab->zErrMsg - ); - } - if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); - sqlite3_free(aBuf); + case 4: /* rawdata */ + case 5: { /* data */ + if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){ + int sz = pCsr->cds.szCompressed; + if( sz>0 ){ + u8 *aBuf = sqlite3_malloc(sz); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + FILE *pFile = zipfileGetFd(pCsr); + rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); + } + if( rc==SQLITE_OK ){ + if( i==5 && pCsr->cds.iCompression ){ + zipfileInflate(ctx, aBuf, sz, pCsr->cds.szUncompressed); + }else{ + sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); + } + sqlite3_free(aBuf); + } } } break; } - case 5: /* method */ + case 6: /* method */ sqlite3_result_int(ctx, pCsr->cds.iCompression); break; } @@ -1073,7 +1155,9 @@ static int zipfileAppendEntry( static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){ const char *z = (const char*)sqlite3_value_text(pVal); int mode = 0; - if( z==0 || (z[0]>=0 && z[0]<=9) ){ + if( z==0 ){ + mode = 33188; /* -rw-r--r-- */ + }else if( z[0]>=0 && z[0]<=9 ){ mode = sqlite3_value_int(pVal); }else{ const char zTemplate[11] = "-rwxrwxrwx"; @@ -1118,8 +1202,8 @@ static int zipfileUpdate( i64 sz; /* Uncompressed size */ const char *zPath; /* Path for new entry */ int nPath; /* strlen(zPath) */ - const u8 *pData; /* Pointer to buffer containing content */ - int nData; /* Size of pData buffer in bytes */ + const u8 *pData = 0; /* Pointer to buffer containing content */ + int nData = 0; /* Size of pData buffer in bytes */ int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ ZipfileCDS cds; /* New Central Directory Structure entry */ @@ -1143,45 +1227,54 @@ static int zipfileUpdate( nPath = (int)strlen(zPath); rc = zipfileGetMode(pTab, apVal[3], &mode); if( rc!=SQLITE_OK ) return rc; - mTime = sqlite3_value_int64(apVal[4]); - sz = sqlite3_value_int(apVal[5]); - pData = sqlite3_value_blob(apVal[6]); - nData = sqlite3_value_bytes(apVal[6]); - - /* If a NULL value is inserted into the 'method' column, do automatic - ** compression. */ - if( nData>0 && sqlite3_value_type(apVal[7])==SQLITE_NULL ){ - pFree = (u8*)sqlite3_malloc(nData); - if( pFree==0 ){ - rc = SQLITE_NOMEM; - }else{ - int res; - z_stream str; - memset(&str, 0, sizeof(str)); - str.next_in = (z_const Bytef*)pData; - str.avail_in = nData; - str.next_out = pFree; - str.avail_out = nData; - deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); - res = deflate(&str, Z_FINISH); - if( res==Z_STREAM_END ){ - pData = pFree; - nData = str.total_out; - iMethod = 8; - }else if( res!=Z_OK ){ - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); - rc = SQLITE_ERROR; - } - deflateEnd(&str); - } + if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){ + mTime = (sqlite3_int64)time(0); }else{ - iMethod = sqlite3_value_int(apVal[7]); + mTime = sqlite3_value_int64(apVal[4]); + } + + if( sqlite3_value_type(apVal[5])==SQLITE_NULL /* sz */ + && sqlite3_value_type(apVal[6])==SQLITE_NULL /* rawdata */ + && sqlite3_value_type(apVal[7])!=SQLITE_NULL /* data */ + ){ + const u8 *aIn = sqlite3_value_blob(apVal[7]); + int nIn = sqlite3_value_bytes(apVal[7]); + int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; + + iMethod = sqlite3_value_int(apVal[8]); + sz = nIn; + if( iMethod!=0 && iMethod!=8 ){ + rc = SQLITE_CONSTRAINT; + }else if( bAuto || iMethod ){ + rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nData); + if( rc==SQLITE_OK ){ + if( iMethod || nData65535 ){ pTab->base.zErrMsg = sqlite3_mprintf( "zipfile: invalid compression method: %d", iMethod ); rc = SQLITE_ERROR; } + }else{ + rc = SQLITE_CONSTRAINT; } if( rc==SQLITE_OK ){ @@ -1352,52 +1445,6 @@ static int zipfileRegister(sqlite3 *db){ # define zipfileRegister(x) SQLITE_OK #endif -/* -** zipfile_uncompress(DATA, SZ, METHOD) -*/ -static void zipfileUncompressFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int iMethod; - - iMethod = sqlite3_value_int(argv[2]); - if( iMethod==0 ){ - sqlite3_result_value(context, argv[0]); - }else if( iMethod==8 ){ - Byte *res; - int sz = sqlite3_value_int(argv[1]); - z_stream str; - memset(&str, 0, sizeof(str)); - str.next_in = (Byte*)sqlite3_value_blob(argv[0]); - str.avail_in = sqlite3_value_bytes(argv[0]); - res = str.next_out = (Byte*)sqlite3_malloc(sz); - if( res==0 ){ - sqlite3_result_error_nomem(context); - }else{ - int err; - str.avail_out = sz; - - err = inflateInit2(&str, -15); - if( err!=Z_OK ){ - zipfileCtxErrorMsg(context, "inflateInit2() failed (%d)", err); - }else{ - err = inflate(&str, Z_NO_FLUSH); - if( err!=Z_STREAM_END ){ - zipfileCtxErrorMsg(context, "inflate() failed (%d)", err); - }else{ - sqlite3_result_blob(context, res, sz, SQLITE_TRANSIENT); - } - } - sqlite3_free(res); - inflateEnd(&str); - } - }else{ - zipfileCtxErrorMsg(context, "unrecognized compression method: %d", iMethod); - } -} - #ifdef _WIN32 __declspec(dllexport) #endif @@ -1406,13 +1453,8 @@ int sqlite3_zipfile_init( char **pzErrMsg, const sqlite3_api_routines *pApi ){ - int rc = SQLITE_OK; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ - rc = sqlite3_create_function(db, "zipfile_uncompress", 3, - SQLITE_UTF8, 0, zipfileUncompressFunc, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; return zipfileRegister(db); } diff --git a/manifest b/manifest index 5f5f829f97..67ccbcf14d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sin\sthe\ssqlite3expert.c\scode\srevealed\sby\s-fsanitize. -D 2018-01-08T17:34:15.620 +C Update\sthe\szipfile\smodule\sso\sthat\sit\smatches\sthe\sdocumentation. +D 2018-01-08T19:59:59.813 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb @@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 8075df9296beeebc344567927d114c6d3201110a29110013388d233fa7d4fb2c +F ext/misc/zipfile.c d576a5b473333f5d14f9380cf005b52a684a1ee46d4a6cfb08802564e6365e5e F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74 -F src/shell.c.in be04d6797b4ee066f064dc370bdcb148c84c6c6979bf12f70025a7a89cf79346 +F src/shell.c.in 2aa65d155202d1caf457cb7112ec47d1aded1bc54b20e7f8f7cf81ca1dbb43bf F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e -F test/zipfile.test a1dd0429294cb9487900fc2b29aa9921329f20a7314aa0921b668246172ac090 +F test/zipfile.test ad4278e1ebb1c7bc0fcd7f9b47df18916b9e8f841165119865a5a6a095a2d0ba F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7182591d351dde22ed2f6a60521d1d7c10a610d702e79693412efc6938167be0 -R 49ecd97a6ce816710af5cab9bccdaf02 +P 7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4 +R 86cc2218634a3d23586f8875ed6e23c5 U dan -Z 0d0d547dd2ee63ac3d70f7968bc046aa +Z 3ef32c645be21defc25c769ce8063d43 diff --git a/manifest.uuid b/manifest.uuid index 0ced8aee6a..930322bb0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4 \ No newline at end of file +7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 9758d5d180..aa92d372ab 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4847,14 +4847,12 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){ const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", - "zipfile_uncompress(data, sz, method)" + "data" }; const char *azSource[] = { "sqlar", "zipfile(?3)" }; - - sqlite3_stmt *pSql = 0; int rc = SQLITE_OK; char *zDir = 0; diff --git a/test/zipfile.test b/test/zipfile.test index 627945651b..4e0c9b18d1 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -28,15 +28,18 @@ do_execsql_test 1.0 { 1 mode {} 0 {} 0 2 mtime {} 0 {} 0 3 sz {} 0 {} 0 - 4 data {} 0 {} 0 - 5 method {} 0 {} 0 + 4 rawdata {} 0 {} 0 + 5 data {} 0 {} 0 + 6 method {} 0 {} 0 } do_execsql_test 1.1.1 { - INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0); + INSERT INTO zz(name, mode, mtime, sz, rawdata, method) + VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0); } do_execsql_test 1.1.2 { - INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0); + INSERT INTO zz(name, mtime, sz, rawdata, method) + VALUES('g.txt', 1000000002, 5, '12345', 0); } do_execsql_test 1.2 { @@ -47,14 +50,13 @@ do_execsql_test 1.2 { } do_execsql_test 1.3 { - INSERT INTO zz VALUES('h.txt', - '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL + INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', + '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb' ); } do_execsql_test 1.4 { - SELECT name, mtime, zipfile_uncompress(data, sz, method), method - FROM zipfile('test.zip'); + SELECT name, mtime, data, method FROM zipfile('test.zip'); } { f.txt 1000000000 abcde 0 g.txt 1000000002 12345 0 @@ -63,7 +65,8 @@ do_execsql_test 1.4 { do_execsql_test 1.5.1 { BEGIN; - INSERT INTO zz VALUES('i.txt', '-rw-r--r--', 1000000006, 5, 'zxcvb', 0); + INSERT INTO zz(name, mode, mtime, data, method) + VALUES('i.txt', '-rw-r--r--', 1000000006, 'zxcvb', 0); SELECT name FROM zz; COMMIT; } {f.txt g.txt h.txt i.txt}