diff --git a/manifest b/manifest index 084b50ff08..6e1234fc69 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sreprepare\sis\sneeded\safter\sbinding\sto\sa\svariable\swith\sa\snumber\slarger\nthan\s32,\sset\sonly\sthe\shigh-order\sbit\sof\sthe\sVdbe.expmask\srather\sthan\ssetting\nall\sbits.\s\sThis\scould\spotentially\sresult\sin\sfewer\sfalse-positive\sreprepares. -D 2017-03-03T21:51:40.099 +C Add\san\sinitial\simplementation\sof\sthe\s"PRAGMA\soptimize"\scommand. +D 2017-03-06T17:33:58.293 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -334,16 +334,16 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 8a2af8a16e4d95ec2327d3f180cb0bab4b2074c1 +F src/analyze.c 6d8234916c29be943e6ea28b5bef67dff98d9905 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 91baade79832becb44d85cd7c39b5ebe170666d8 -F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac +F src/btree.c e2bae0a03f73d119910fb35c9550987564065137 +F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 -F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f +F src/build.c 43f903c9082040ced2b421543cb0300c2973647d F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 @@ -390,8 +390,8 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c b127edeb54c744a101b371cfa2e221fd741bcd72 -F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a +F src/pragma.c 75fdb05838c303cf0ce5846ca011b8103531c42c +F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -402,7 +402,7 @@ F src/shell.c 27d2b31099fd2cd749e44d025ef9b54ca26692cb F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h a23e18aebdd0d851c2956a74a3a4f12ff202b472 +F src/sqliteInt.h df268ce1d04df042cf43b557d2309eb0b71e86c4 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -464,7 +464,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 83f387d9e6842b1dc99f6e85bb577c5bbc4e397d +F src/vdbe.c f520378e510fd36bbf289921798dbc8f2b3dc30d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 @@ -478,8 +478,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 01baf58b72f3ddb7793cdee2871f751e3e09b35e -F src/whereInt.h c0b092180f04608d80c258174b0a14a1f9c8d02f +F src/where.c 1a3a8adb717a20f17c186f3baa22b0b5f3a5ab13 +F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -524,6 +524,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f +F test/autoanalyze1.test 1ba80d5e1412d46503566b70741a5eea060da929 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1499,7 +1500,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl ebb4bfcd2f8010e0a3934b6118db4b5f2f5edf5c +F tool/mkpragmatab.tcl 2ffe6d5fdc2d3381621d6c77978ba054466e757f F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1561,7 +1562,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4a04c48a311b19ba5e566877dc5baff543c41aba -R 637ce5713001d3d9ffad62c9c4caff88 +P 45797feefe90cb7da53256b0c42fdaa1221d0a27 5f7fc79aa06ca9b79664c50c3c277c98a74ff9a0 +R d076238fa06701bd917c02e95912d525 +T +closed 5f7fc79aa06ca9b79664c50c3c277c98a74ff9a0 U drh -Z 435b5c3ba7edfd2dba5c4b61417da7dd +Z 60608bb3fd59af1c228de2bc78db17f1 diff --git a/manifest.uuid b/manifest.uuid index 9ee7804d61..c68e3d13a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45797feefe90cb7da53256b0c42fdaa1221d0a27 \ No newline at end of file +137aeb2b160888100bc1e871b00860149e5f6196 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 098b6c7e37..d1df000943 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1388,27 +1388,14 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( i==1 ) continue; /* Do not analyze the TEMP database */ analyzeDatabase(pParse, i); } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ - analyzeTable(pParse, pIdx->pTable, pIdx); - }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ - analyzeTable(pParse, pTab, 0); - } - sqlite3DbFree(db, z); - } - } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); }else{ - /* Form 3: Analyze the fully qualified table name */ + /* Form 3: Analyze the table or index named as an argument */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zDbSName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -1418,10 +1405,11 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } sqlite3DbFree(db, z); } - } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -1550,7 +1538,11 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; @@ -1559,6 +1551,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; } return 0; @@ -1853,15 +1846,20 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ HashElem *i; char *zSql; int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); /* Clear any prior statistics */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - pIdx->aiRowLogEst[0] = 0; + pIdx->hasStat1 = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; @@ -1884,9 +1882,9 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); } /* Load the statistics from the sqlite_stat4 table. */ @@ -1896,7 +1894,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bDisable--; } - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; diff --git a/src/btree.c b/src/btree.c index e942dbce03..b99cab89ec 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5368,6 +5368,30 @@ int sqlite3BtreeEof(BtCursor *pCur){ return (CURSOR_VALID!=pCur->eState); } +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1; + + for(n=1, i=0; i<=pCur->iPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor diff --git a/src/btree.h b/src/btree.h index ae57468e3f..0e017f5300 100644 --- a/src/btree.h +++ b/src/btree.h @@ -296,6 +296,7 @@ u32 sqlite3BtreePayloadSize(BtCursor*); char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); +i64 sqlite3BtreeRowCountEst(BtCursor*); #ifndef SQLITE_OMIT_INCRBLOB int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); diff --git a/src/build.c b/src/build.c index 6e1e2177be..2ce7c90747 100644 --- a/src/build.c +++ b/src/build.c @@ -3455,6 +3455,9 @@ void sqlite3DefaultRowEst(Index *pIdx){ int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + /* Set the first entry (number of rows in the index) to the estimated ** number of rows in the table, or half the number of rows in the table ** for a partial index. But do not let the estimate drop below 10. */ diff --git a/src/pragma.c b/src/pragma.c index 151a7a2e82..c5f2a77eef 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1114,29 +1114,33 @@ void sqlite3Pragma( } break; +#ifdef SQLITE_DEBUG case PragTyp_STATS: { Index *pIdx; HashElem *i; - pParse->nMem = 4; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", + sqlite3VdbeMultiLoad(v, 1, "ssiii", pTab->zName, 0, pTab->szTabRow, - pTab->nRowLogEst); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pTab->nRowLogEst, + pTab->tabFlags); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", + sqlite3VdbeMultiLoad(v, 2, "siii", pIdx->zName, pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; +#endif case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; @@ -1858,6 +1862,118 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA optimize + ** PRAGMA optimize(MASK) + ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) + ** + ** Attempt to optimize the database. All schemas are optimized in the first + ** two forms, and only the specified schema is optimized in the latter two. + ** + ** The details of optimizations performed by this pragma does are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** The optional argument is a bitmask of optimizations to perform: + ** + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. + ** + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is 0x000e, which means perform all of the optimizations + ** listed above except do not set Debug Mode. New optimizations may be + ** added in future releases but they will be turned off by default. The + ** default MASK will always be 0x0e. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of + ** the following are true: + ** + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (3) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. + */ + case PragTyp_OPTIMIZE: { + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xe; + } + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; /* Always analyze if any index lacks statistics */ + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } + } + sqlite3VdbeAddOp0(v, OP_Expire); + break; + } + /* ** PRAGMA busy_timeout ** PRAGMA busy_timeout = N diff --git a/src/pragma.h b/src/pragma.h index 5d8d0aa35b..9b1c723b3e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -30,11 +30,11 @@ #define PragTyp_LOCKING_MODE 22 #define PragTyp_PAGE_COUNT 23 #define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 +#define PragTyp_OPTIMIZE 25 +#define PragTyp_PAGE_SIZE 26 +#define PragTyp_SECURE_DELETE 27 +#define PragTyp_SHRINK_MEMORY 28 +#define PragTyp_SOFT_HEAP_LIMIT 29 #define PragTyp_SYNCHRONOUS 30 #define PragTyp_TABLE_INFO 31 #define PragTyp_TEMP_STORE 32 @@ -48,6 +48,7 @@ #define PragTyp_REKEY 40 #define PragTyp_LOCK_STATUS 41 #define PragTyp_PARSER_TRACE 42 +#define PragTyp_STATS 43 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -71,47 +72,48 @@ static const char *const pragCName[] = { /* 4 */ "notnull", /* 5 */ "dflt_value", /* 6 */ "pk", - /* 7 */ "table", /* Used by: stats */ - /* 8 */ "index", - /* 9 */ "width", - /* 10 */ "height", - /* 11 */ "seqno", /* Used by: index_info */ - /* 12 */ "cid", - /* 13 */ "name", - /* 14 */ "seqno", /* Used by: index_xinfo */ - /* 15 */ "cid", - /* 16 */ "name", - /* 17 */ "desc", - /* 18 */ "coll", - /* 19 */ "key", - /* 20 */ "seq", /* Used by: index_list */ - /* 21 */ "name", - /* 22 */ "unique", - /* 23 */ "origin", - /* 24 */ "partial", - /* 25 */ "seq", /* Used by: database_list */ - /* 26 */ "name", - /* 27 */ "file", - /* 28 */ "seq", /* Used by: collation_list */ - /* 29 */ "name", - /* 30 */ "id", /* Used by: foreign_key_list */ - /* 31 */ "seq", - /* 32 */ "table", - /* 33 */ "from", - /* 34 */ "to", - /* 35 */ "on_update", - /* 36 */ "on_delete", - /* 37 */ "match", - /* 38 */ "table", /* Used by: foreign_key_check */ - /* 39 */ "rowid", - /* 40 */ "parent", - /* 41 */ "fkid", - /* 42 */ "busy", /* Used by: wal_checkpoint */ - /* 43 */ "log", - /* 44 */ "checkpointed", - /* 45 */ "timeout", /* Used by: busy_timeout */ - /* 46 */ "database", /* Used by: lock_status */ - /* 47 */ "status", + /* 7 */ "tbl", /* Used by: stats */ + /* 8 */ "idx", + /* 9 */ "wdth", + /* 10 */ "hght", + /* 11 */ "flgs", + /* 12 */ "seqno", /* Used by: index_info */ + /* 13 */ "cid", + /* 14 */ "name", + /* 15 */ "seqno", /* Used by: index_xinfo */ + /* 16 */ "cid", + /* 17 */ "name", + /* 18 */ "desc", + /* 19 */ "coll", + /* 20 */ "key", + /* 21 */ "seq", /* Used by: index_list */ + /* 22 */ "name", + /* 23 */ "unique", + /* 24 */ "origin", + /* 25 */ "partial", + /* 26 */ "seq", /* Used by: database_list */ + /* 27 */ "name", + /* 28 */ "file", + /* 29 */ "seq", /* Used by: collation_list */ + /* 30 */ "name", + /* 31 */ "id", /* Used by: foreign_key_list */ + /* 32 */ "seq", + /* 33 */ "table", + /* 34 */ "from", + /* 35 */ "to", + /* 36 */ "on_update", + /* 37 */ "on_delete", + /* 38 */ "match", + /* 39 */ "table", /* Used by: foreign_key_check */ + /* 40 */ "rowid", + /* 41 */ "parent", + /* 42 */ "fkid", + /* 43 */ "busy", /* Used by: wal_checkpoint */ + /* 44 */ "log", + /* 45 */ "checkpointed", + /* 46 */ "timeout", /* Used by: busy_timeout */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", }; /* Definitions of all built-in pragmas */ @@ -157,7 +159,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 45, 1, + /* ColNames: */ 46, 1, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "cache_size", @@ -194,7 +196,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "collation_list", /* ePragTyp: */ PragTyp_COLLATION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 28, 2, + /* ColNames: */ 29, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) @@ -229,7 +231,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, - /* ColNames: */ 25, 3, + /* ColNames: */ 26, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) @@ -266,14 +268,14 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "foreign_key_check", /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 38, 4, + /* ColNames: */ 39, 4, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) {/* zName: */ "foreign_key_list", /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 30, 8, + /* ColNames: */ 31, 8, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -336,17 +338,17 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 11, 3, + /* ColNames: */ 12, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 20, 5, + /* ColNames: */ 21, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 14, 6, + /* ColNames: */ 15, 6, /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) @@ -393,7 +395,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "lock_status", /* ePragTyp: */ PragTyp_LOCK_STATUS, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 46, 2, + /* ColNames: */ 47, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -412,6 +414,13 @@ static const PragmaName aPragmaName[] = { /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_Result1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, @@ -510,11 +519,11 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_SqlTrace }, #endif #endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 7, 4, + /* ColNames: */ 7, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -593,7 +602,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 42, 3, + /* ColNames: */ 43, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -604,4 +613,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 60 on by default, 74 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d9ec8c5bc9..e95d63ec35 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1333,6 +1333,7 @@ struct sqlite3 { u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 skipBtreeMutex; /* True if no shared-cache backends */ + u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ @@ -1883,12 +1884,13 @@ struct Table { #define TF_Ephemeral 0x0002 /* An ephemeral table */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ -/* available for reuse: 0x0010 */ +#define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ #define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ -#define TF_HasNotNull 0x0100 /* Contains NOT NULL constraints */ - +#define TF_StatsUsed 0x0100 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ +#define TF_HasNotNull 0x0200 /* Contains NOT NULL constraints */ /* ** Test to see whether or not a table is a virtual table. This is @@ -2131,6 +2133,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/src/vdbe.c b/src/vdbe.c index fabe98ae57..5901ba0eb5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4847,6 +4847,33 @@ case OP_Last: { /* jump */ break; } +/* Opcode: IfSmaller P1 P2 P3 * * +** +** Estimate the number of rows in the table P1. Jump to P2 if that +** estimate is less than approximately 2**(0.1*P3). +*/ +case OP_IfSmaller: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + i64 sz; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + sz = sqlite3BtreeRowCountEst(pCrsr); + if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + } + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: SorterSort P1 P2 * * * ** @@ -5491,6 +5518,18 @@ case OP_CreateTable: { /* out2 */ break; } +/* Opcode: SqlExec * * * P4 * +** +** Run the SQL statement or statements specified in the P4 string. +*/ +case OP_SqlExec: { + db->nSqlExec++; + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + db->nSqlExec--; + if( rc ) goto abort_due_to_error; + break; +} + /* Opcode: ParseSchema P1 * * P4 * ** ** Read and parse all entries from the SQLITE_MASTER table of database P1 diff --git a/src/where.c b/src/where.c index f2bf400a2d..8d0dbb0979 100644 --- a/src/where.c +++ b/src/where.c @@ -2387,6 +2387,11 @@ static int whereLoopAddBtreeIndex( continue; } + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + }else{ + pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->u.btree.nBtm = saved_nBtm; @@ -2934,7 +2939,15 @@ static int whereLoopAddBtree( } } + pBuilder->bldFlags = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_StatsUsed; + } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; diff --git a/src/whereInt.h b/src/whereInt.h index c6195f53ed..f065fae6ba 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -395,8 +395,13 @@ struct WhereLoopBuilder { UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif + unsigned int bldFlags; /* SQLITE_BLDF_* flags */ }; +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test new file mode 100644 index 0000000000..77e767a89e --- /dev/null +++ b/test/autoanalyze1.test @@ -0,0 +1,123 @@ +# 2017-02-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for the logic used to estimate when +# running ANALYZE would be beneficial. +# +# Note that this test uses some hard-coded bitmask values from sqliteInt.h. +# If any of the following constants changes: +# +# define TF_HasStat1 0x0010 +# define TF_StatsUsed 0x0100 +# +# then some of the magic numbers in test results below might need to be +# adjusted. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# There is nothing to test if ANALYZE is disable for this build. +# These tests also use "PRAGMA stats" which are only enabled for +# debugging builds. +# +ifcapable {!debug || !analyze} { + finish_test + return +} + +do_execsql_test autoanalyze1-100 { + -- Build up a test table with some indexes + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE UNIQUE INDEX t1bc ON t1(b,c); + CREATE INDEX t1d ON t1(d); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b,c,d) SELECT x, x, x, x FROM c; + -- Verify that the hasStat1 flag is clear on on indexes + SELECT idx, flgs FROM pragma_stats + WHERE idx IS NOT NULL + ORDER BY idx; + -- Verify that the TF_HasStat1 flag is clear on the table + SELECT tbl, (flgs & 0x10)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {t1bc 0 t1d 0 t1 0} + +# No use of stat1 recorded so far +do_execsql_test autoanalyze1-110 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Access using a unique index does not set the TF_StatsUsed flag. +# +do_execsql_test autoanalyze1-200 { + SELECT * FROM t1 WHERE a=55; +} {55 55 55 55} +do_execsql_test autoanalyze1-201 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-210 { + SELECT * FROM t1 WHERE a IN (55,199,299); +} {55 55 55 55} +do_execsql_test autoanalyze1-211 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-220 { + SELECT * FROM t1 WHERE (b,c)=(45,45); +} {45 45 45 45} +do_execsql_test autoanalyze1-221 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Any use of the non-unique t1d index triggers the use of stats. +# +sqlite3 db test.db +do_execsql_test autoanalyze1-300 { + SELECT * FROM t1 WHERE d=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-301 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +sqlite3 db test.db +do_execsql_test autoanalyze1-310 { + SELECT * FROM t1 WHERE d=45 AND a=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-311 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} ;# The ROWID lookup short-circuits the d=45 constraint + +sqlite3 db test.db +do_execsql_test autoanalyze1-320 { + SELECT * FROM t1 WHERE d=45 AND a IN (45,46); +} {45 45 45 45} +do_execsql_test autoanalyze1-321 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# Any use of prefix of a unique index triggers the use of stats +# +sqlite3 db test.db +do_execsql_test autoanalyze1-400 { + SELECT * FROM t1 WHERE b=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-401 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# The TF_StatsUsed flag is reset when the database is reopened +# +sqlite3 db test.db +do_execsql_test autoanalyze1-500 { + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +finish_test diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index c22f72d491..59b245cc76 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,8 +225,8 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height - IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + COLS: tbl idx wdth hght flgs + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info TYPE: INDEX_INFO @@ -361,6 +361,9 @@ set pragma_def { NAME: threads FLAG: Result0 + + NAME: optimize + FLAG: Result1 } # Open the output file