diff --git a/doc/lemon.html b/doc/lemon.html index 114526f372..f05c481d79 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -23,6 +23,26 @@ or embedded controllers.

This document is an introduction to the Lemon parser generator.

+

Security Note

+ +

The language parser code created by Lemon is very robust and +is well-suited for use in internet-facing applications that need to +safely process maliciously crafted inputs. + +

The "lemon.exe" command-line tool itself works great when given a valid +input grammar file and almost always gives helpful +error messages for malformed inputs. However, it is possible for +a malicious user to craft a grammar file that will cause +lemon.exe to crash. +We do not see this as a problem, as lemon.exe is not intended to be used +with hostile inputs. +To summarize:

+ + +

Theory of Operation

The main goal of Lemon is to translate a context free grammar (CFG) diff --git a/ext/misc/series.c b/ext/misc/series.c index 9f80bb46ca..684995f3b6 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -33,7 +33,7 @@ ** The generate_series "function" is really a virtual table with the ** following schema: ** -** CREATE FUNCTION generate_series( +** CREATE TABLE generate_series( ** value, ** start HIDDEN, ** stop HIDDEN, diff --git a/manifest b/manifest index 7924724845..83b166c1c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sinterfaces\ssqlite3_prepare_v3()\sand\ssqlite3_prepare16_v3()\swith\sthe\nextra\sprepFlags\sargument.\s\sAdd\sthe\sSQLITE_PREPARE_PERSISTENT\soption\sas\sone\nbit\sin\sthat\sargument. -D 2017-06-01T00:54:35.692 +C Merge\sin\strunk\senhancements. +D 2017-06-07T16:25:25.048 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -33,7 +33,7 @@ F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 F configure 1bcc61cdd063171d8945551c265e5701a770deeff77e0ad634f8d22e4e91c831 x F configure.ac 13f45f02e6c51dd0e347315b5401c3f047712b7f79b7f35619115c23755afcff F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad -F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 +F doc/lemon.html 1f8b8d4c9f5cfe40e679fee279cc9eb2da8e6eb74ad406028538d7864cc4b6cb F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd @@ -228,7 +228,7 @@ F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/remember.c 8440f8d0b452c5cdefb62b57135ccd1267aa729d F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb -F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 +F ext/misc/series.c b0f5f346aca9b7ff7caaf0da2efb4ad462441abd4dcd92a460cb573b3ea2370b F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -351,13 +351,13 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 8ac6ae352c63998228718b5f11faa0da2676710898a47284de78e96cb41dca35 F src/btree.h 3edc5329bc59534d2d15b4f069a9f54b779a7e51289e98fa481ae3c0e526a5ca F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610 -F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f +F src/build.c 88a8cdc11d1c081ed565aa3e795bdf9160f4556463b4c4555e9860b59dd80340 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c F src/date.c cc42a41c7422389860d40419a5e3bce5eaf6e7835c3ba2677751dc653550a5c7 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d -F src/delete.c bc2bfc227002ebe091e7cc321f09a48e52f86e1cd5ff9b028009db0671556749 +F src/delete.c 3213547e97b676c6fa79948b7a9ede4801ea04a01a2043241deafedf132ecf5d F src/expr.c 452c6f3aa656aabf3eefe96bb5f316b2c987fbc12c647964e4ed880f193ca31f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c db65492ae549c3b548c9ef1f279ce1684f1c473b116e1c56a90878cd5dcf968d @@ -367,7 +367,7 @@ F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c ea52232babcee65f1f9833d0fefc3b598662a004c19a178ef4fc0981e2f02a58 +F src/insert.c 974499a3999d339a4c1ba8ef129a988d9f136b3789e423808b38cdc19d28adbe F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9 F src/main.c 1054e4dbb4fcca84246ed48b35164a996f5a52daa56c275157a5d583c0e2cd00 @@ -406,7 +406,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c adf3ef9843135b1383321ad751f16f5a40c3f37925154555a3e61653d2a954e8 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d93205e43af302d9eb147fddecc956509ee9d0dde6297ee3f93c591f60f0e295 -F src/shell.c 3f761fe604174b31aacd2ea2eacef5e6fe550111d60c0d71532cc008c68cf3f3 +F src/shell.c eca7e7fe8dae859aa56e462c5b35c735976fa1e5e1d7a90fd5a32aa4615c1825 F src/sqlite.h.in e644e75c21ab6247ebd8bca07a999d7d108fe8be00adc14d8c4b6004f1e75c34 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 @@ -434,7 +434,7 @@ F src/test_config.c edcba290248dc18736dd814c9b95863c6762e0b35753048d8cbe5bf65f7a F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 -F src/test_fs.c e16cbe68d3b107e00a907c20a9a02629870eb69b +F src/test_fs.c 35a2f7dd8a915900873386331386d9ba1ae1b5026d74fd20c2807bc76221f291 F src/test_func.c a4fdab3363b436c1b12660e9362ce3f3782b7b5e F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 @@ -472,7 +472,7 @@ F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c cce462ad3cf1cad1944e105f773712a979e23fbe302328dc2885b0f4a612e1f6 +F src/vdbe.c 6783778df820f3e0c68289fe5d12ddd34168b0eb1acf248668fec6e2ea1012d5 F src/vdbe.h dde459b1e8a02b8445ecfd5959f38cd5ebb6b0ad392d491d8b159ac8193d231a F src/vdbeInt.h ba7a9436196e693cc51532138ba9431af2716efbc1afe62b03b2724a67853764 F src/vdbeapi.c 899d8f021c89ab348708b3a9b00b855f5ecc3c0f949a75359a61a3c621021281 @@ -488,7 +488,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c d46044e7a5842560dfe7122d93ff5145dd4a96f4d0bf5ba5910a7731b8c01e79 F src/where.c 67f98714b07ec3c1d5e033a63d23c0fd70c24861b7b46b69b10700f22dca6ffe F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 -F src/wherecode.c 8ad48867660519e262a401720845dc76934f86f558ec9606335fafcd7a2554f8 +F src/wherecode.c 339ee802d9d311acf0cba8b5a9a092e167ef71c3a777d4b3e57de25d193251c7 F src/whereexpr.c a2fe3811d45af45a5c6667caabc15e01054fe6228c64e86e1f7d2ba5ef5284f9 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -925,7 +925,7 @@ F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb2848 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 +F test/kvtest.c d2b8cfc91047ebf6cac4f3a04f19c3a864e4ecfd683bbb65c395df450b8dc79c F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1416,7 +1416,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test 5f9253eb9e41ba9fe94f4aa3e9230191893d7764 +F test/vtabH.test 26d54e8b5407f797638b787a55f9c88323850a58dd142de02d06b9a1159bd283 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 @@ -1582,10 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c26cf978eead1c9d265eddabaa421e7735b472fcf2792cd2bdeb0901bcf3fb44 -R 920bebc856e573465ecc651ded3907f5 -T *branch * prepare_v3 -T *sym-prepare_v3 * -T -sym-trunk * +P 4a25c5883380fe5990d8180adb58c3bdc7a3d081bc4c69cd4de3cd57074fb251 234ede26e30f20e6c33002739ed8be35dbfb5c77700bd857ff31072b9b7df347 +R 0957dad5bbe072099826cf9a4a9a3885 U drh -Z a2627d28422eb06f03507d6701dbaec5 +Z 6991b6c9f5798221e32381bb8967907a diff --git a/manifest.uuid b/manifest.uuid index 0149bb39c2..da9d04a97c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a25c5883380fe5990d8180adb58c3bdc7a3d081bc4c69cd4de3cd57074fb251 \ No newline at end of file +f8bbb608cbf6c245628e3d362e9181fb3dc402b1d8241bcb687caf395f63c916 \ No newline at end of file diff --git a/src/build.c b/src/build.c index e04406d857..c4bb0300a5 100644 --- a/src/build.c +++ b/src/build.c @@ -939,7 +939,11 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nTabRef = 1; +#ifdef SQLITE_DEFAULT_ROWEST + pTable->nRowLogEst = sqlite3LogEst(SQLITE_DEFAULT_ROWEST); +#else pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); +#endif assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; diff --git a/src/delete.c b/src/delete.c index 6bb77b67b4..9ffcf7e58e 100644 --- a/src/delete.c +++ b/src/delete.c @@ -459,7 +459,7 @@ void sqlite3DeleteFrom( sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEphCur, iKey, iPk, nPk); }else{ /* Add the rowid of the row to be deleted to the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ + nKey = 1; /* OP_DeferredSeek always uses a single rowid */ sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); } } diff --git a/src/insert.c b/src/insert.c index 48502b7e07..e684eaafa9 100644 --- a/src/insert.c +++ b/src/insert.c @@ -524,6 +524,7 @@ void sqlite3Insert( if( pParse->nErr || db->mallocFailed ){ goto insert_cleanup; } + dest.iSDParm = 0; /* Suppress a harmless compiler warning */ /* If the Select object is really just a simple VALUES() list with a ** single row (the common case) then keep that one row of values diff --git a/src/shell.c b/src/shell.c index 322c6e5c4f..6a436da488 100644 --- a/src/shell.c +++ b/src/shell.c @@ -509,6 +509,18 @@ static int strlen30(const char *z){ return 0x3fffffff & (int)(z2 - z); } +/* +** Return the length of a string in characters. Multibyte UTF8 characters +** count as a single character. +*/ +static int strlenChar(const char *z){ + int n = 0; + while( *z ){ + if( (0xc0&*(z++))!=0x80 ) n++; + } + return n; +} + /* ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer @@ -1917,9 +1929,9 @@ static int shell_callback( w = 0; } if( w==0 ){ - w = strlen30(azCol[i] ? azCol[i] : ""); + w = strlenChar(azCol[i] ? azCol[i] : ""); if( w<10 ) w = 10; - n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullValue); + n = strlenChar(azArg && azArg[i] ? azArg[i] : p->nullValue); if( wactualWidth) ){ @@ -1954,8 +1966,8 @@ static int shell_callback( }else{ w = 10; } - if( p->cMode==MODE_Explain && azArg[i] && strlen30(azArg[i])>w ){ - w = strlen30(azArg[i]); + if( p->cMode==MODE_Explain && azArg[i] && strlenChar(azArg[i])>w ){ + w = strlenChar(azArg[i]); } if( i==1 && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ diff --git a/src/test_fs.c b/src/test_fs.c index dd590a6623..8192beb99b 100644 --- a/src/test_fs.c +++ b/src/test_fs.c @@ -545,6 +545,7 @@ static int fstreeFilter( zDir = zQuery; } } + if( nDir==0 ) nDir = 1; sqlite3_bind_text(pCsr->pStmt, 1, zDir, nDir, SQLITE_TRANSIENT); sqlite3_bind_text(pCsr->pStmt, 2, zRoot, nRoot, SQLITE_TRANSIENT); diff --git a/src/vdbe.c b/src/vdbe.c index 9a0678ced1..4a412b4e50 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2497,7 +2497,9 @@ case OP_Column: { pC = p->apCsr[pOp->p1]; p2 = pOp->p2; - /* If the cursor cache is stale, bring it up-to-date */ + /* If the cursor cache is stale (meaning it is not currently point at + ** the correct row) then bring it up-to-date by doing the necessary + ** B-Tree seek. */ rc = sqlite3VdbeCursorMoveto(&pC, &p2); if( rc ) goto abort_due_to_error; @@ -5265,8 +5267,8 @@ case OP_IdxDelete: { break; } -/* Opcode: Seek P1 * P3 P4 * -** Synopsis: Move P3 to P1.rowid +/* Opcode: DeferredSeek P1 * P3 P4 * +** Synopsis: Move P3 to P1.rowid if needed ** ** P1 is an open index cursor and P3 is a cursor on the corresponding ** table. This opcode does a deferred seek of the P3 table cursor @@ -5293,11 +5295,11 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_Seek: -case OP_IdxRowid: { /* out2 */ - VdbeCursor *pC; /* The P1 index cursor */ - VdbeCursor *pTabCur; /* The P2 table cursor (OP_Seek only) */ - i64 rowid; /* Rowid that P1 current points to */ +case OP_DeferredSeek: +case OP_IdxRowid: { /* out2 */ + VdbeCursor *pC; /* The P1 index cursor */ + VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ + i64 rowid; /* Rowid that P1 current points to */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -5323,7 +5325,7 @@ case OP_IdxRowid: { /* out2 */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( pOp->opcode==OP_Seek ){ + if( pOp->opcode==OP_DeferredSeek ){ assert( pOp->p3>=0 && pOp->p3nCursor ); pTabCur = p->apCsr[pOp->p3]; assert( pTabCur!=0 ); diff --git a/src/wherecode.c b/src/wherecode.c index feda45f911..7d1ea38596 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -966,10 +966,10 @@ static void codeCursorHint( ** ** Normally, this is just: ** -** OP_Seek $iCur $iRowid +** OP_DeferredSeek $iCur $iRowid ** ** However, if the scan currently being coded is a branch of an OR-loop and -** the statement currently being coded is a SELECT, then P3 of the OP_Seek +** the statement currently being coded is a SELECT, then P3 of OP_DeferredSeek ** is set to iIdxCur and P4 is set to point to an array of integers ** containing one entry for each column of the table cursor iCur is open ** on. For each table column, if the column is the i'th column of the @@ -988,7 +988,7 @@ static void codeDeferredSeek( assert( iIdxCur>0 ); assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); - sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur); + sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur); if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) ){ diff --git a/test/kvtest.c b/test/kvtest.c index 4fa733f1a7..b80adfea09 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -71,14 +71,21 @@ static const char zHelp[] = "\n" " --variance V Randomly vary M by plus or minus V\n" "\n" -" kvtest export DBFILE DIRECTORY\n" +" kvtest export DBFILE DIRECTORY [--tree]\n" "\n" " Export all the blobs in the kv table of DBFILE into separate\n" -" files in DIRECTORY.\n" +" files in DIRECTORY. DIRECTORY is created if it does not previously\n" +" exist. If the --tree option is used, then the blobs are written\n" +" into a hierarchy of directories, using names like 00/00/00,\n" +" 00/00/01, 00/00/02, and so forth. Without the --tree option, all\n" +" files are in the top-level directory with names like 000000, 000001,\n" +" 000002, and so forth.\n" "\n" -" kvtest stat DBFILE\n" +" kvtest stat DBFILE [options]\n" "\n" -" Display summary information about DBFILE\n" +" Display summary information about DBFILE. Options:\n" +"\n" +" --vacuum Run VACUUM on the database file\n" "\n" " kvtest run DBFILE [options]\n" "\n" @@ -90,12 +97,18 @@ static const char zHelp[] = " --cache-size N Database cache size\n" " --count N Read N blobs\n" " --desc Read blobs in descending order\n" +" --fsync Synchronous file writes\n" +" --integrity-check Run \"PRAGMA integrity_check\" after test\n" " --max-id N Maximum blob key to use\n" " --mmap N Mmap as much as N bytes of DBFILE\n" +" --multitrans Each read or write in its own transaction\n" +" --nocheckpoint Omit the checkpoint on WAL mode writes\n" +" --nosync Set \"PRAGMA synchronous=OFF\"\n" " --jmode MODE Set MODE journal mode prior to starting\n" " --random Read blobs in a random order\n" " --start N Start reading with this blob key\n" " --stats Output operating stats before exiting\n" +" --update Do an overwrite test\n" ; /* Reference resources used */ @@ -111,6 +124,7 @@ static const char zHelp[] = # include #else /* Provide Windows equivalent for the needed parts of unistd.h */ +# include # include # define R_OK 2 # define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) @@ -118,6 +132,31 @@ static const char zHelp[] = # define access _access #endif +#include + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on +** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). +** So we have to define the macros in different ways depending on the +** compiler. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(__PTRDIFF_TYPE__)(X)) +#else +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(intptr_t)(X)) +#endif /* ** Show thqe help text and quit. @@ -201,13 +240,23 @@ static int integerValue(const char *zArg){ /* ** Check the filesystem object zPath. Determine what it is: ** -** PATH_DIR A directory +** PATH_DIR A single directory holding many files +** PATH_TREE A directory hierarchy with files at the leaves ** PATH_DB An SQLite database ** PATH_NEXIST Does not exist ** PATH_OTHER Something else +** +** PATH_DIR means all of the separate files are grouped together +** into a single directory with names like 000000, 000001, 000002, and +** so forth. PATH_TREE means there is a hierarchy of directories so +** that no single directory has too many entries. The files have names +** like 00/00/00, 00/00/01, 00/00/02 and so forth. The decision between +** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory +** named "00" at the top-level. */ #define PATH_DIR 1 -#define PATH_DB 2 +#define PATH_TREE 2 +#define PATH_DB 3 #define PATH_NEXIST 0 #define PATH_OTHER 99 static int pathType(const char *zPath){ @@ -217,7 +266,15 @@ static int pathType(const char *zPath){ memset(&x, 0, sizeof(x)); rc = stat(zPath, &x); if( rc<0 ) return PATH_OTHER; - if( S_ISDIR(x.st_mode) ) return PATH_DIR; + if( S_ISDIR(x.st_mode) ){ + char *zLayer1 = sqlite3_mprintf("%s/00", zPath); + memset(&x, 0, sizeof(x)); + rc = stat(zLayer1, &x); + sqlite3_free(zLayer1); + if( rc<0 ) return PATH_DIR; + if( S_ISDIR(x.st_mode) ) return PATH_TREE; + return PATH_DIR; + } if( (x.st_size%512)==0 ) return PATH_DB; return PATH_OTHER; } @@ -328,6 +385,7 @@ static int statMain(int argc, char **argv){ sqlite3 *db; char *zSql; sqlite3_stmt *pStmt; + int doVacuum = 0; assert( strcmp(argv[1],"stat")==0 ); assert( argc>=3 ); @@ -336,12 +394,21 @@ static int statMain(int argc, char **argv){ char *z = argv[i]; if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z); if( z[1]=='-' ) z++; + if( strcmp(z, "-vacuum")==0 ){ + doVacuum = 1; + continue; + } fatalError("unknown option: \"%s\"", argv[i]); } rc = sqlite3_open(zDb, &db); if( rc ){ fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); } + if( doVacuum ){ + printf("Vacuuming...."); fflush(stdout); + sqlite3_exec(db, "VACUUM", 0, 0, 0); + printf(" done\n"); + } zSql = sqlite3_mprintf( "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))" " FROM kv" @@ -374,39 +441,53 @@ static int statMain(int argc, char **argv){ printf("Page-count: %8d\n", sqlite3_column_int(pStmt, 0)); } sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf("PRAGMA freelist_count"); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); + sqlite3_free(zSql); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("Freelist-count: %8d\n", sqlite3_column_int(pStmt, 0)); + } + sqlite3_finalize(pStmt); + rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check(10)", -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare integrity check: %s", sqlite3_errmsg(db)); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("Integrity-check: %s\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); sqlite3_close(db); return 0; } /* -** Implementation of the "writefile(X,Y)" SQL function. The argument Y -** is written into file X. The number of bytes written is returned. Or -** NULL is returned if something goes wrong, such as being unable to open -** file X for writing. +** remember(V,PTR) +** +** Return the integer value V. Also save the value of V in a +** C-language variable whose address is PTR. */ -static void writefileFunc( - sqlite3_context *context, +static void rememberFunc( + sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ - FILE *out; - const char *z; - sqlite3_int64 rc; - const char *zFile; + sqlite3_int64 v; + sqlite3_int64 ptr; + assert( argc==2 ); + v = sqlite3_value_int64(argv[0]); + ptr = sqlite3_value_int64(argv[1]); + *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v; + sqlite3_result_int64(pCtx, v); +} - zFile = (const char*)sqlite3_value_text(argv[0]); - if( zFile==0 ) return; - out = fopen(zFile, "wb"); - if( out==0 ) return; - z = (const char*)sqlite3_value_blob(argv[1]); - if( z==0 ){ - rc = 0; - }else{ - rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); - } - fclose(out); - printf("\r%s ", zFile); fflush(stdout); - sqlite3_result_int64(context, rc); +/* +** Make sure a directory named zDir exists. +*/ +static void kvtest_mkdir(const char *zDir){ +#if defined(_WIN32) + (void)mkdir(zDir); +#else + (void)mkdir(zDir, 0755); +#endif } /* @@ -416,32 +497,77 @@ static int exportMain(int argc, char **argv){ char *zDb; char *zDir; sqlite3 *db; - char *zSql; + sqlite3_stmt *pStmt; int rc; - char *zErrMsg = 0; + int ePathType; + int nFN; + char *zFN; + char *zTail; + size_t nWrote; + int i; assert( strcmp(argv[1],"export")==0 ); assert( argc>=3 ); + if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]"); zDb = argv[2]; - if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY"); zDir = argv[3]; - if( pathType(zDir)!=PATH_DIR ){ + kvtest_mkdir(zDir); + for(i=4; i