mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhancements to deserialize: (1) Add the SQLITE_FCNTL_SIZE_LIMIT file control

Browse Source 
to set a maximum size for an in-memory database, defaulting to 
SQLITE_MEMDB_DEFAULT_MAXSIZE or 1GiB.  (2) Honor the SQLITE_DESERIALIZE_READONLY
flag. (3) Enhance the TCL interface to support -maxsize N and -readonly BOOLEAN.
(4) Add the --maxsize option to the ".open" command and on the command-line for
the CLI.

FossilOrigin-Name: 30f08d58882819a69e353bcc1b6b349664bbfbe00aa1c115ba44a9fd899fcc5b
This commit is contained in:
drh 2019-01-22 16:06:20 +00:00
parent 247c1b4a0b
commit 6ca644818b
7 changed files with 140 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
manifest
View File

@ -1,5 +1,5 @@
C Make\ssure\scursors\sare\sopened\son\sall\sindexes\sfor\san\sUPDATE\sOR\sREPLACE\nregardless\sof\swhether\sor\snot\sthe\sindexes\sare\spartial\sor\scontain\scolumns\nthat\smight\sneed\sto\sbe\supdated.
D 2019-01-22T13:45:48.673
C Enhancements\sto\sdeserialize:\s(1)\sAdd\sthe\sSQLITE_FCNTL_SIZE_LIMIT\sfile\scontrol\nto\sset\sa\smaximum\ssize\sfor\san\sin-memory\sdatabase,\sdefaulting\sto\s\nSQLITE_MEMDB_DEFAULT_MAXSIZE\sor\s1GiB.\s\s(2)\sHonor\sthe\sSQLITE_DESERIALIZE_READONLY\nflag.\s(3)\sEnhance\sthe\sTCL\sinterface\sto\ssupport\s-maxsize\sN\sand\s-readonly\sBOOLEAN.\n(4)\sAdd\sthe\s--maxsize\soption\sto\sthe\s".open"\scommand\sand\son\sthe\scommand-line\sfor\nthe\sCLI.
D 2019-01-22T16:06:20.120
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in 0e7c107ebcaff26681bc5bcf017557db85aa828d6f7fd652d748b7a78072c298
@ -485,7 +485,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
F src/memdb.c cb4013d56fa71c79c498717cbc47b27dd1c7653fd866584b2071ae04114eec46
F src/memdb.c 25d36740e40ed3f3758c17bd6ed7db099a6b1d8033d7ea7058496774ac14d8c4
F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661
F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
F src/mutex.c bae36f8af32c22ad80bbf0ccebec63c252b6a2b86e4d3e42672ff287ebf4a604
@ -515,15 +515,15 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
F src/resolve.c a40867ce07a9b58121d6f9a8fc969555d3c9bdcb6c2b5fc202670815af8dbd91
F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
F src/select.c f7260c833c87c52ac187bc160ccc675a67d5a226cacd7eb1cdcb3c3ff25bde76
F src/shell.c.in 58b94d2473d84f457dfee94bd0dac3173d39dfdfad058c1c4042a157ec43c4fa
F src/sqlite.h.in b54cd42d2f3b739a00de540cafe2dcd0de3b8e1748a2db33a68def487e9e602f
F src/shell.c.in 7649cd10a1ad8bf4caf78e501db97e96bb4c4f35120b8dd2c3f983e859ea1325
F src/sqlite.h.in 8ded85ecaa768afd196b24201382ccdf00e5bab6861e30549cd750bebd273a0b
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
F src/sqliteInt.h a2330a569d8c5461aa35fe3ad29a1885e13ddfd07088a3e833131490c3a99ca9
F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
F src/tclsqlite.c e72862a271348d779672b45a730c33fd0c535e630ff927e8ce4a0c908d1d28c6
F src/tclsqlite.c 6b19e7562195aaf881f3e35e2472dc01ae3cb156961db5126c3d616744729b7e
F src/test1.c 64cdc914a77102e008dfae7adaa4ded54c2d4953d1464ea8709805a2aab755eb
F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
@ -1120,7 +1120,7 @@ F test/malloctraceviewer.tcl b7a54595270c1d201abf1c3f3d461f27eaf24cdef623ad08a0f
F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e
F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f
F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7
F test/memdb1.test 61aa1dbdeea6320791d2ff42a9a6149d5716be674bf06ee0ffa0aad1bf3eb5f8
F test/memdb1.test 0632e6ea56c48e3c6e9b0c73e120310bad8f93762543f809e267888f5a37943f
F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e
F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08
@ -1801,7 +1801,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P ba3b8412726548a0716c1a2d67260c3b7e31956474f4cd4ce607cf2cebc667dd
R 7ce2d510eb9d051132c78a15e023756d
P e148cdad35520e6684cfeba23b003f60b55f83a6bf621aff16be8aa5612cdcee
R 9d6d2798c27f659933ba85e19f8b3d4e
U drh
Z d4e1fff084d7bc0803e5ba0a13f2d0e8
Z aaf32ec586902c86687ff793f750210d

2
manifest.uuid
View File

@ -1 +1 @@
e148cdad35520e6684cfeba23b003f60b55f83a6bf621aff16be8aa5612cdcee
30f08d58882819a69e353bcc1b6b349664bbfbe00aa1c115ba44a9fd899fcc5b

38
src/memdb.c
View File

@ -34,13 +34,19 @@ typedef struct MemFile MemFile;
struct MemFile {
sqlite3_file base; /* IO methods */
sqlite3_int64 sz; /* Size of the file */
sqlite3_int64 szMax; /* Space allocated to aData */
sqlite3_int64 szAlloc; /* Space allocated to aData */
sqlite3_int64 szMax; /* Maximum allowed size of the file */
unsigned char *aData; /* content of the file */
int nMmap; /* Number of memory mapped pages */
unsigned mFlags; /* Flags */
int eLock; /* Most recent lock against this file */
};
/* The default maximum size of an in-memory database */
#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
#endif
/*
** Methods for MemFile
*/
@ -160,10 +166,15 @@ static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){
if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){
return SQLITE_FULL;
}
if( newSz>p->szMax ){
return SQLITE_FULL;
}
newSz *= 2;
if( newSz>p->szMax ) newSz = p->szMax;
pNew = sqlite3_realloc64(p->aData, newSz);
if( pNew==0 ) return SQLITE_NOMEM;
p->aData = pNew;
p->szMax = newSz;
p->szAlloc = newSz;
return SQLITE_OK;
}
@ -177,10 +188,11 @@ static int memdbWrite(
sqlite_int64 iOfst
){
MemFile *p = (MemFile *)pFile;
if( p->mFlags & SQLITE_DESERIALIZE_READONLY ) return SQLITE_READONLY;
if( iOfst+iAmt>p->sz ){
int rc;
if( iOfst+iAmt>p->szMax
&& (rc = memdbEnlarge(p, (iOfst+iAmt)*2))!=SQLITE_OK
if( iOfst+iAmt>p->szAlloc
&& (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK
){
return rc;
}
@ -250,6 +262,19 @@ static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){
*(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz);
rc = SQLITE_OK;
}
if( op==SQLITE_FCNTL_SIZE_LIMIT ){
sqlite3_int64 iLimit = *(sqlite3_int64*)pArg;
if( iLimit<p->sz ){
if( iLimit<0 ){
iLimit = p->szMax;
}else{
iLimit = p->sz;
}
}
p->szMax = iLimit;
*(sqlite3_int64*)pArg = iLimit;
rc = SQLITE_OK;
}
return rc;
}
@ -311,6 +336,7 @@ static int memdbOpen(
assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */
*pOutFlags = flags | SQLITE_OPEN_MEMORY;
p->base.pMethods = &memdb_io_methods;
p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
return SQLITE_OK;
}
@ -560,7 +586,11 @@ int sqlite3_deserialize(
}else{
p->aData = pData;
p->sz = szDb;
p->szAlloc = szBuf;
p->szMax = szBuf;
if( p->szMax<SQLITE_MEMDB_DEFAULT_MAXSIZE ){
p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
}
p->mFlags = mFlags;
rc = SQLITE_OK;
}

18
src/shell.c.in
View File

@ -1025,6 +1025,7 @@ struct ShellState {
int showHeader; /* True to show column names in List or Column mode */
int nCheck; /* Number of ".check" commands run */
unsigned shellFlgs; /* Various flags */
sqlite3_int64 szMax; /* --maxsize argument to .open */
char *zDestTable; /* Name of destination table when MODE_Insert */
char *zTempFile; /* Temporary file that might need deleting */
char zTestcase[30]; /* Name of current test case */
@ -3449,6 +3450,7 @@ static const char *(azHelp[]) = {
#ifdef SQLITE_ENABLE_DESERIALIZE
" --deserialize Load into memory useing sqlite3_deserialize()",
" --hexdb Load the output of \"dbtotxt\" as an in-memory database",
" --maxsize N Maximum size for --hexdb or --deserialized database",
#endif
" --new Initialize FILE to an empty database",
" --readonly Open FILE readonly",
@ -3927,6 +3929,9 @@ static void open_db(ShellState *p, int openFlags){
if( rc ){
utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
}
if( p->szMax>0 ){
sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
}
}
#endif
}
@ -6841,6 +6846,7 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(p->zFreeOnClose);
p->zFreeOnClose = 0;
p->openMode = SHELL_OPEN_UNSPEC;
p->szMax = 0;
/* Check for command-line arguments */
for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
const char *z = azArg[iName];
@ -6859,6 +6865,8 @@ static int do_meta_command(char *zLine, ShellState *p){
p->openMode = SHELL_OPEN_DESERIALIZE;
}else if( optionMatch(z, "hexdb") ){
p->openMode = SHELL_OPEN_HEXDB;
}else if( optionMatch(z, "maxsize") && iName+1<nArg ){
p->szMax = integerValue(azArg[++iName]);
#endif /* SQLITE_ENABLE_DESERIALIZE */
}else if( z[0]=='-' ){
utf8_printf(stderr, "unknown option: %s\n", z);
@ -8549,6 +8557,9 @@ static const char zOptions[] =
" -column set output mode to 'column'\n"
" -cmd COMMAND run \"COMMAND\" before reading stdin\n"
" -csv set output mode to 'csv'\n"
#if defined(SQLITE_ENABLE_DESERIALIZE)
" -deserialize open the database using sqlite3_deserialize()\n"
#endif
" -echo print commands before execution\n"
" -init FILENAME read/process named file\n"
" -[no]header turn headers on or off\n"
@ -8561,6 +8572,9 @@ static const char zOptions[] =
" -line set output mode to 'line'\n"
" -list set output mode to 'list'\n"
" -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n"
#if defined(SQLITE_ENABLE_DESERIALIZE)
" -maxsize N maximum size for a --deserialize database\n"
#endif
" -mmap N default mmap size set to N\n"
#ifdef SQLITE_ENABLE_MULTIPLEX
" -multiplex enable the multiplexor VFS\n"
@ -8871,6 +8885,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
#ifdef SQLITE_ENABLE_DESERIALIZE
}else if( strcmp(z,"-deserialize")==0 ){
data.openMode = SHELL_OPEN_DESERIALIZE;
}else if( strcmp(z,"-maxsize")==0 && i+1<argc ){
data.szMax = integerValue(argv[++i]);
#endif
}else if( strcmp(z,"-readonly")==0 ){
data.openMode = SHELL_OPEN_READONLY;
@ -8972,6 +8988,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
#ifdef SQLITE_ENABLE_DESERIALIZE
}else if( strcmp(z,"-deserialize")==0 ){
data.openMode = SHELL_OPEN_DESERIALIZE;
}else if( strcmp(z,"-maxsize")==0 && i+1<argc ){
data.szMax = integerValue(argv[++i]);
#endif
}else if( strcmp(z,"-readonly")==0 ){
data.openMode = SHELL_OPEN_READONLY;

10
src/sqlite.h.in
View File

@ -823,6 +823,15 @@ struct sqlite3_io_methods {
** file space based on this hint in order to help writes to the database
** file run faster.
**
** <li>[[SQLITE_FCNTL_SIZE_LIMIT]]
** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that
** implements [sqlite3_deserialize()] to set an upper bound on the size
** of the in-memory database. The argument is a pointer to a [sqlite3_int64].
** If the integer pointed to is negative, then it is filled in with the
** current limit. Otherwise the limit is set to the larger of the value
** of the integer pointed to and the current database size. The integer
** pointed to is set to the new limit.
**
** <li>[[SQLITE_FCNTL_CHUNK_SIZE]]
** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS
** extends and truncates the database file in chunks of a size specified
@ -1131,6 +1140,7 @@ struct sqlite3_io_methods {
#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33
#define SQLITE_FCNTL_LOCK_TIMEOUT 34
#define SQLITE_FCNTL_DATA_VERSION 35
#define SQLITE_FCNTL_SIZE_LIMIT 36
/* deprecated names */
#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE

53
src/tclsqlite.c
View File

@ -2418,7 +2418,7 @@ static int SQLITE_TCLAPI DbObjCmd(
}
/*
** $db deserialize ?DATABASE? VALUE
** $db deserialize ?-maxsize N? ?-readonly BOOL? ?DATABASE? VALUE
**
** Reopen DATABASE (default "main") using the content in $VALUE
*/
@ -2428,38 +2428,65 @@ static int SQLITE_TCLAPI DbObjCmd(
(char*)0);
rc = TCL_ERROR;
#else
const char *zSchema;
Tcl_Obj *pValue;
const char *zSchema = 0;
Tcl_Obj *pValue = 0;
unsigned char *pBA;
unsigned char *pData;
int len, xrc;
sqlite3_int64 mxSize = 0;
int i;
int isReadonly = 0;
if( objc==3 ){
zSchema = 0;
pValue = objv[2];
}else if( objc==4 ){
zSchema = Tcl_GetString(objv[2]);
pValue = objv[3];
}else{
if( objc<3 ){
Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? VALUE");
rc = TCL_ERROR;
break;
}
for(i=2; i<objc-1; i++){
const char *z = Tcl_GetString(objv[i]);
if( strcmp(z,"-maxsize")==0 && i<objc-2 ){
rc = Tcl_GetWideIntFromObj(interp, objv[++i], &mxSize);
if( rc ) goto deserialize_error;
continue;
}
if( strcmp(z,"-readonly")==0 && i<objc-2 ){
rc = Tcl_GetBooleanFromObj(interp, objv[++i], &isReadonly);
if( rc ) goto deserialize_error;
continue;
}
if( zSchema==0 && i==objc-2 && z[0]!='-' ){
zSchema = z;
continue;
}
Tcl_AppendResult(interp, "unknown option: ", z, (char*)0);
rc = TCL_ERROR;
goto deserialize_error;
}
pValue = objv[objc-1];
pBA = Tcl_GetByteArrayFromObj(pValue, &len);
pData = sqlite3_malloc64( len );
if( pData==0 && len>0 ){
Tcl_AppendResult(interp, "out of memory", (char*)0);
rc = TCL_ERROR;
}else{
int flags;
if( len>0 ) memcpy(pData, pBA, len);
xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len,
SQLITE_DESERIALIZE_FREEONCLOSE |
SQLITE_DESERIALIZE_RESIZEABLE);
if( isReadonly ){
flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY;
}else{
flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE;
}
xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len, flags);
if( xrc ){
Tcl_AppendResult(interp, "unable to set MEMDB content", (char*)0);
rc = TCL_ERROR;
}
if( mxSize>0 ){
sqlite3_file_control(pDb->db, zSchema,SQLITE_FCNTL_SIZE_LIMIT,&mxSize);
}
}
deserialize_error:
#endif
break;
}

27
test/memdb1.test
View File

@ -72,6 +72,31 @@ do_execsql_test 140 {
PRAGMA page_count;
} {2}
do_test 150 {
catch {db deserialize -unknown 1 $db1} msg
set msg
} {unknown option: -unknown}
do_test 151 {
db deserialize -readonly 1 $db1
db eval {SELECT * FROM t1}
} {1 2}
do_test 152 {
catchsql {INSERT INTO t1 VALUES(3,4);}
} {1 {attempt to write a readonly database}}
breakpoint
do_test 160 {
db deserialize -maxsize 32768 $db1
db eval {SELECT * FROM t1}
} {1 2}
do_test 161 {
db eval {INSERT INTO t1 VALUES(3,4); SELECT * FROM t1}
} {1 2 3 4}
do_test 162 {
catchsql {INSERT INTO t1 VALUES(5,randomblob(100000))}
} {1 {database or disk is full}}
# Build a largish on-disk database and serialize it. Verify that the
# serialization works.
#
@ -154,7 +179,7 @@ do_test 600 {
do_test 610 {
set rc [catch {db deserialize a b c} msg]
lappend rc $msg
} {1 {wrong # args: should be "db deserialize ?DATABASE? VALUE"}}
} {1 {unknown option: a}}
do_test 620 {
set rc [catch {db serialize a b} msg]
lappend rc $msg