diff --git a/manifest b/manifest index b5a208b609..c29935bcfc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\scursors\sare\sopened\son\sall\sindexes\sfor\san\sUPDATE\sOR\sREPLACE\nregardless\sof\swhether\sor\snot\sthe\sindexes\sare\spartial\sor\scontain\scolumns\nthat\smight\sneed\sto\sbe\supdated. -D 2019-01-22T13:45:48.673 +C Enhancements\sto\sdeserialize:\s(1)\sAdd\sthe\sSQLITE_FCNTL_SIZE_LIMIT\sfile\scontrol\nto\sset\sa\smaximum\ssize\sfor\san\sin-memory\sdatabase,\sdefaulting\sto\s\nSQLITE_MEMDB_DEFAULT_MAXSIZE\sor\s1GiB.\s\s(2)\sHonor\sthe\sSQLITE_DESERIALIZE_READONLY\nflag.\s(3)\sEnhance\sthe\sTCL\sinterface\sto\ssupport\s-maxsize\sN\sand\s-readonly\sBOOLEAN.\n(4)\sAdd\sthe\s--maxsize\soption\sto\sthe\s".open"\scommand\sand\son\sthe\scommand-line\sfor\nthe\sCLI. +D 2019-01-22T16:06:20.120 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 0e7c107ebcaff26681bc5bcf017557db85aa828d6f7fd652d748b7a78072c298 @@ -485,7 +485,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memdb.c cb4013d56fa71c79c498717cbc47b27dd1c7653fd866584b2071ae04114eec46 +F src/memdb.c 25d36740e40ed3f3758c17bd6ed7db099a6b1d8033d7ea7058496774ac14d8c4 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 F src/mutex.c bae36f8af32c22ad80bbf0ccebec63c252b6a2b86e4d3e42672ff287ebf4a604 @@ -515,15 +515,15 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c a40867ce07a9b58121d6f9a8fc969555d3c9bdcb6c2b5fc202670815af8dbd91 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c f7260c833c87c52ac187bc160ccc675a67d5a226cacd7eb1cdcb3c3ff25bde76 -F src/shell.c.in 58b94d2473d84f457dfee94bd0dac3173d39dfdfad058c1c4042a157ec43c4fa -F src/sqlite.h.in b54cd42d2f3b739a00de540cafe2dcd0de3b8e1748a2db33a68def487e9e602f +F src/shell.c.in 7649cd10a1ad8bf4caf78e501db97e96bb4c4f35120b8dd2c3f983e859ea1325 +F src/sqlite.h.in 8ded85ecaa768afd196b24201382ccdf00e5bab6861e30549cd750bebd273a0b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 F src/sqliteInt.h a2330a569d8c5461aa35fe3ad29a1885e13ddfd07088a3e833131490c3a99ca9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 -F src/tclsqlite.c e72862a271348d779672b45a730c33fd0c535e630ff927e8ce4a0c908d1d28c6 +F src/tclsqlite.c 6b19e7562195aaf881f3e35e2472dc01ae3cb156961db5126c3d616744729b7e F src/test1.c 64cdc914a77102e008dfae7adaa4ded54c2d4953d1464ea8709805a2aab755eb F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 @@ -1120,7 +1120,7 @@ F test/malloctraceviewer.tcl b7a54595270c1d201abf1c3f3d461f27eaf24cdef623ad08a0f F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 -F test/memdb1.test 61aa1dbdeea6320791d2ff42a9a6149d5716be674bf06ee0ffa0aad1bf3eb5f8 +F test/memdb1.test 0632e6ea56c48e3c6e9b0c73e120310bad8f93762543f809e267888f5a37943f F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 @@ -1801,7 +1801,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba3b8412726548a0716c1a2d67260c3b7e31956474f4cd4ce607cf2cebc667dd -R 7ce2d510eb9d051132c78a15e023756d +P e148cdad35520e6684cfeba23b003f60b55f83a6bf621aff16be8aa5612cdcee +R 9d6d2798c27f659933ba85e19f8b3d4e U drh -Z d4e1fff084d7bc0803e5ba0a13f2d0e8 +Z aaf32ec586902c86687ff793f750210d diff --git a/manifest.uuid b/manifest.uuid index e35c40c4db..012ba884b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e148cdad35520e6684cfeba23b003f60b55f83a6bf621aff16be8aa5612cdcee \ No newline at end of file +30f08d58882819a69e353bcc1b6b349664bbfbe00aa1c115ba44a9fd899fcc5b \ No newline at end of file diff --git a/src/memdb.c b/src/memdb.c index 7723d40525..c6603a1c81 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -34,13 +34,19 @@ typedef struct MemFile MemFile; struct MemFile { sqlite3_file base; /* IO methods */ sqlite3_int64 sz; /* Size of the file */ - sqlite3_int64 szMax; /* Space allocated to aData */ + sqlite3_int64 szAlloc; /* Space allocated to aData */ + sqlite3_int64 szMax; /* Maximum allowed size of the file */ unsigned char *aData; /* content of the file */ int nMmap; /* Number of memory mapped pages */ unsigned mFlags; /* Flags */ int eLock; /* Most recent lock against this file */ }; +/* The default maximum size of an in-memory database */ +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824 +#endif + /* ** Methods for MemFile */ @@ -160,10 +166,15 @@ static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){ if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){ return SQLITE_FULL; } + if( newSz>p->szMax ){ + return SQLITE_FULL; + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; pNew = sqlite3_realloc64(p->aData, newSz); if( pNew==0 ) return SQLITE_NOMEM; p->aData = pNew; - p->szMax = newSz; + p->szAlloc = newSz; return SQLITE_OK; } @@ -177,10 +188,11 @@ static int memdbWrite( sqlite_int64 iOfst ){ MemFile *p = (MemFile *)pFile; + if( p->mFlags & SQLITE_DESERIALIZE_READONLY ) return SQLITE_READONLY; if( iOfst+iAmt>p->sz ){ int rc; - if( iOfst+iAmt>p->szMax - && (rc = memdbEnlarge(p, (iOfst+iAmt)*2))!=SQLITE_OK + if( iOfst+iAmt>p->szAlloc + && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK ){ return rc; } @@ -250,6 +262,19 @@ static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); rc = SQLITE_OK; } + if( op==SQLITE_FCNTL_SIZE_LIMIT ){ + sqlite3_int64 iLimit = *(sqlite3_int64*)pArg; + if( iLimitsz ){ + if( iLimit<0 ){ + iLimit = p->szMax; + }else{ + iLimit = p->sz; + } + } + p->szMax = iLimit; + *(sqlite3_int64*)pArg = iLimit; + rc = SQLITE_OK; + } return rc; } @@ -311,6 +336,7 @@ static int memdbOpen( assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */ *pOutFlags = flags | SQLITE_OPEN_MEMORY; p->base.pMethods = &memdb_io_methods; + p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE; return SQLITE_OK; } @@ -560,7 +586,11 @@ int sqlite3_deserialize( }else{ p->aData = pData; p->sz = szDb; + p->szAlloc = szBuf; p->szMax = szBuf; + if( p->szMaxszMax = SQLITE_MEMDB_DEFAULT_MAXSIZE; + } p->mFlags = mFlags; rc = SQLITE_OK; } diff --git a/src/shell.c.in b/src/shell.c.in index 84c9cab189..ac2ee1981b 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1025,6 +1025,7 @@ struct ShellState { int showHeader; /* True to show column names in List or Column mode */ int nCheck; /* Number of ".check" commands run */ unsigned shellFlgs; /* Various flags */ + sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ char zTestcase[30]; /* Name of current test case */ @@ -3449,6 +3450,7 @@ static const char *(azHelp[]) = { #ifdef SQLITE_ENABLE_DESERIALIZE " --deserialize Load into memory useing sqlite3_deserialize()", " --hexdb Load the output of \"dbtotxt\" as an in-memory database", + " --maxsize N Maximum size for --hexdb or --deserialized database", #endif " --new Initialize FILE to an empty database", " --readonly Open FILE readonly", @@ -3927,6 +3929,9 @@ static void open_db(ShellState *p, int openFlags){ if( rc ){ utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); } + if( p->szMax>0 ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); + } } #endif } @@ -6841,6 +6846,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(p->zFreeOnClose); p->zFreeOnClose = 0; p->openMode = SHELL_OPEN_UNSPEC; + p->szMax = 0; /* Check for command-line arguments */ for(iName=1; iNameopenMode = SHELL_OPEN_DESERIALIZE; }else if( optionMatch(z, "hexdb") ){ p->openMode = SHELL_OPEN_HEXDB; + }else if( optionMatch(z, "maxsize") && iName+1szMax = integerValue(azArg[++iName]); #endif /* SQLITE_ENABLE_DESERIALIZE */ }else if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); @@ -8549,6 +8557,9 @@ static const char zOptions[] = " -column set output mode to 'column'\n" " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -deserialize open the database using sqlite3_deserialize()\n" +#endif " -echo print commands before execution\n" " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" @@ -8561,6 +8572,9 @@ static const char zOptions[] = " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" " -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n" +#if defined(SQLITE_ENABLE_DESERIALIZE) + " -maxsize N maximum size for a --deserialize database\n" +#endif " -mmap N default mmap size set to N\n" #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" @@ -8871,6 +8885,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifdef SQLITE_ENABLE_DESERIALIZE }else if( strcmp(z,"-deserialize")==0 ){ data.openMode = SHELL_OPEN_DESERIALIZE; + }else if( strcmp(z,"-maxsize")==0 && i+1[[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** **
  • [[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified @@ -1131,6 +1140,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 #define SQLITE_FCNTL_LOCK_TIMEOUT 34 #define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 3982ead7be..eb3bedf9d4 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2418,7 +2418,7 @@ static int SQLITE_TCLAPI DbObjCmd( } /* - ** $db deserialize ?DATABASE? VALUE + ** $db deserialize ?-maxsize N? ?-readonly BOOL? ?DATABASE? VALUE ** ** Reopen DATABASE (default "main") using the content in $VALUE */ @@ -2428,38 +2428,65 @@ static int SQLITE_TCLAPI DbObjCmd( (char*)0); rc = TCL_ERROR; #else - const char *zSchema; - Tcl_Obj *pValue; + const char *zSchema = 0; + Tcl_Obj *pValue = 0; unsigned char *pBA; unsigned char *pData; int len, xrc; - - if( objc==3 ){ - zSchema = 0; - pValue = objv[2]; - }else if( objc==4 ){ - zSchema = Tcl_GetString(objv[2]); - pValue = objv[3]; - }else{ + sqlite3_int64 mxSize = 0; + int i; + int isReadonly = 0; + + + if( objc<3 ){ Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? VALUE"); rc = TCL_ERROR; break; } + for(i=2; i0 ){ Tcl_AppendResult(interp, "out of memory", (char*)0); rc = TCL_ERROR; }else{ + int flags; if( len>0 ) memcpy(pData, pBA, len); - xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len, - SQLITE_DESERIALIZE_FREEONCLOSE | - SQLITE_DESERIALIZE_RESIZEABLE); + if( isReadonly ){ + flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY; + }else{ + flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE; + } + xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len, flags); if( xrc ){ Tcl_AppendResult(interp, "unable to set MEMDB content", (char*)0); rc = TCL_ERROR; } + if( mxSize>0 ){ + sqlite3_file_control(pDb->db, zSchema,SQLITE_FCNTL_SIZE_LIMIT,&mxSize); + } } +deserialize_error: #endif break; } diff --git a/test/memdb1.test b/test/memdb1.test index 771d5e42fa..3d1a1f9a63 100644 --- a/test/memdb1.test +++ b/test/memdb1.test @@ -72,6 +72,31 @@ do_execsql_test 140 { PRAGMA page_count; } {2} +do_test 150 { + catch {db deserialize -unknown 1 $db1} msg + set msg +} {unknown option: -unknown} +do_test 151 { + db deserialize -readonly 1 $db1 + db eval {SELECT * FROM t1} +} {1 2} +do_test 152 { + catchsql {INSERT INTO t1 VALUES(3,4);} +} {1 {attempt to write a readonly database}} + +breakpoint +do_test 160 { + db deserialize -maxsize 32768 $db1 + db eval {SELECT * FROM t1} +} {1 2} +do_test 161 { + db eval {INSERT INTO t1 VALUES(3,4); SELECT * FROM t1} +} {1 2 3 4} +do_test 162 { + catchsql {INSERT INTO t1 VALUES(5,randomblob(100000))} +} {1 {database or disk is full}} + + # Build a largish on-disk database and serialize it. Verify that the # serialization works. # @@ -154,7 +179,7 @@ do_test 600 { do_test 610 { set rc [catch {db deserialize a b c} msg] lappend rc $msg -} {1 {wrong # args: should be "db deserialize ?DATABASE? VALUE"}} +} {1 {unknown option: a}} do_test 620 { set rc [catch {db serialize a b} msg] lappend rc $msg