From 33bec3f5e89787125b56243ab7de2fe9353b8b8a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 13:38:15 +0000 Subject: [PATCH 01/12] Enhance the Index and Table objects so that they remember if their stats come from the sqlite_stat1 table. Make the "PRAGMA stats" an SQLITE_DEBUG only pragma. Add the flags column to "PRAGMA stats". These are all preliminary steps toward a "PRAGMA analyze_ifneeded;" feature. FossilOrigin-Name: 85026c8ee143bbd46565660fff8346ef81421546 --- manifest | 25 +++++---- manifest.uuid | 2 +- src/analyze.c | 22 ++++++-- src/build.c | 3 + src/pragma.c | 18 +++--- src/pragma.h | 129 ++++++++++++++++++++++--------------------- src/sqliteInt.h | 18 +++--- tool/mkpragmatab.tcl | 4 +- 8 files changed, 121 insertions(+), 100 deletions(-) diff --git a/manifest b/manifest index dc0eaa5c63..04051afae1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sthat\swas\smade\sto\sfail\sby\sthe\sLIKE\soptimization\senhancement\nin\scheck-in\s[158290c0ab]\sbut\swhich\swent\sunnoticed\sbecause\stest\sbuilds\swere\nrunning\swith\sICU\senabled\sand\sICU\sdisables\sthe\sLIKE\soptimization. -D 2017-02-17T02:04:31.068 +C Enhance\sthe\sIndex\sand\sTable\sobjects\sso\sthat\sthey\sremember\sif\stheir\sstats\scome\nfrom\sthe\ssqlite_stat1\stable.\s\sMake\sthe\s"PRAGMA\sstats"\san\sSQLITE_DEBUG\sonly\npragma.\s\sAdd\sthe\sflags\scolumn\sto\s"PRAGMA\sstats".\s\sThese\sare\sall\spreliminary\nsteps\stoward\sa\s"PRAGMA\sanalyze_ifneeded;"\sfeature. +D 2017-02-17T13:38:15.256 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c ac7a5d7e3cee07d074697904e00e4a8ab7b2b4f5 +F src/analyze.c 1b7197d619788353437d390de42a9bccbb4aa2ac F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -340,7 +340,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 -F src/build.c 9e799f1edd910dfa8a0bc29bd390d35d310596af +F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c a9984df73898c042a5cfc8f9d8e7723d02bc35c9 @@ -387,8 +387,8 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 7831956012f5d764761fbd023e59b0ffc08f5e8d -F src/pragma.h 61aa5389118594bebb28120a6720401aee34ce1a +F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 +F src/pragma.h cea24a631982fd1a26fcddd46f596d22303b4247 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 46fe8e5aee3825d77fa771216ef263dc947030e7 +F src/sqliteInt.h 54bc20855f2a1a99a1b726f1384b5ce9ab67a0ff F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1494,7 +1494,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl ebb4bfcd2f8010e0a3934b6118db4b5f2f5edf5c +F tool/mkpragmatab.tcl 9c0a855e0daf83e54ffd9fd438260cd0c92f25d0 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1556,7 +1556,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8a03be1dc42737ba0712d33f639ea26dc243b20e -R b29803fdd4fd514ba7baa53a58b26311 +P 218b2bbb0de07288889f6762d4461ea8acd78969 +R af8aeffdf276070e8e8912906300a1a4 +T *branch * auto-analyze +T *sym-auto-analyze * +T -sym-trunk * U drh -Z af65459984ca8c135fabce3c62812860 +Z f8720207819c88fb362c87de069d287a diff --git a/manifest.uuid b/manifest.uuid index d642d67abe..fd9de00f55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -218b2bbb0de07288889f6762d4461ea8acd78969 \ No newline at end of file +85026c8ee143bbd46565660fff8346ef81421546 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 890ae7c3b2..3ecf469cd3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1532,7 +1532,11 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; @@ -1541,6 +1545,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; } return 0; @@ -1835,15 +1840,20 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ HashElem *i; char *zSql; int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); /* Clear any prior statistics */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - pIdx->aiRowLogEst[0] = 0; + pIdx->hasStat1 = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; @@ -1866,9 +1876,9 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); } /* Load the statistics from the sqlite_stat4 table. */ @@ -1878,7 +1888,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bDisable--; } - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; diff --git a/src/build.c b/src/build.c index cd9c81be82..f85c55f0a1 100644 --- a/src/build.c +++ b/src/build.c @@ -3454,6 +3454,9 @@ void sqlite3DefaultRowEst(Index *pIdx){ int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + /* Set the first entry (number of rows in the index) to the estimated ** number of rows in the table, or half the number of rows in the table ** for a partial index. But do not let the estimate drop below 10. */ diff --git a/src/pragma.c b/src/pragma.c index b1775a4082..f28ff9d13c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1098,29 +1098,33 @@ void sqlite3Pragma( } break; +#ifdef SQLITE_DEBUG case PragTyp_STATS: { Index *pIdx; HashElem *i; - pParse->nMem = 4; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", + sqlite3VdbeMultiLoad(v, 1, "ssiii", pTab->zName, 0, pTab->szTabRow, - pTab->nRowLogEst); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pTab->nRowLogEst, + pTab->tabFlags); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", + sqlite3VdbeMultiLoad(v, 2, "siii", pIdx->zName, pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; +#endif case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; diff --git a/src/pragma.h b/src/pragma.h index 5d8d0aa35b..2a96ecaa43 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -34,20 +34,20 @@ #define PragTyp_SECURE_DELETE 26 #define PragTyp_SHRINK_MEMORY 27 #define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 -#define PragTyp_SYNCHRONOUS 30 -#define PragTyp_TABLE_INFO 31 -#define PragTyp_TEMP_STORE 32 -#define PragTyp_TEMP_STORE_DIRECTORY 33 -#define PragTyp_THREADS 34 -#define PragTyp_WAL_AUTOCHECKPOINT 35 -#define PragTyp_WAL_CHECKPOINT 36 -#define PragTyp_ACTIVATE_EXTENSIONS 37 -#define PragTyp_HEXKEY 38 -#define PragTyp_KEY 39 -#define PragTyp_REKEY 40 -#define PragTyp_LOCK_STATUS 41 -#define PragTyp_PARSER_TRACE 42 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 +#define PragTyp_STATS 42 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -75,43 +75,44 @@ static const char *const pragCName[] = { /* 8 */ "index", /* 9 */ "width", /* 10 */ "height", - /* 11 */ "seqno", /* Used by: index_info */ - /* 12 */ "cid", - /* 13 */ "name", - /* 14 */ "seqno", /* Used by: index_xinfo */ - /* 15 */ "cid", - /* 16 */ "name", - /* 17 */ "desc", - /* 18 */ "coll", - /* 19 */ "key", - /* 20 */ "seq", /* Used by: index_list */ - /* 21 */ "name", - /* 22 */ "unique", - /* 23 */ "origin", - /* 24 */ "partial", - /* 25 */ "seq", /* Used by: database_list */ - /* 26 */ "name", - /* 27 */ "file", - /* 28 */ "seq", /* Used by: collation_list */ - /* 29 */ "name", - /* 30 */ "id", /* Used by: foreign_key_list */ - /* 31 */ "seq", - /* 32 */ "table", - /* 33 */ "from", - /* 34 */ "to", - /* 35 */ "on_update", - /* 36 */ "on_delete", - /* 37 */ "match", - /* 38 */ "table", /* Used by: foreign_key_check */ - /* 39 */ "rowid", - /* 40 */ "parent", - /* 41 */ "fkid", - /* 42 */ "busy", /* Used by: wal_checkpoint */ - /* 43 */ "log", - /* 44 */ "checkpointed", - /* 45 */ "timeout", /* Used by: busy_timeout */ - /* 46 */ "database", /* Used by: lock_status */ - /* 47 */ "status", + /* 11 */ "flags", + /* 12 */ "seqno", /* Used by: index_info */ + /* 13 */ "cid", + /* 14 */ "name", + /* 15 */ "seqno", /* Used by: index_xinfo */ + /* 16 */ "cid", + /* 17 */ "name", + /* 18 */ "desc", + /* 19 */ "coll", + /* 20 */ "key", + /* 21 */ "seq", /* Used by: index_list */ + /* 22 */ "name", + /* 23 */ "unique", + /* 24 */ "origin", + /* 25 */ "partial", + /* 26 */ "seq", /* Used by: database_list */ + /* 27 */ "name", + /* 28 */ "file", + /* 29 */ "seq", /* Used by: collation_list */ + /* 30 */ "name", + /* 31 */ "id", /* Used by: foreign_key_list */ + /* 32 */ "seq", + /* 33 */ "table", + /* 34 */ "from", + /* 35 */ "to", + /* 36 */ "on_update", + /* 37 */ "on_delete", + /* 38 */ "match", + /* 39 */ "table", /* Used by: foreign_key_check */ + /* 40 */ "rowid", + /* 41 */ "parent", + /* 42 */ "fkid", + /* 43 */ "busy", /* Used by: wal_checkpoint */ + /* 44 */ "log", + /* 45 */ "checkpointed", + /* 46 */ "timeout", /* Used by: busy_timeout */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", }; /* Definitions of all built-in pragmas */ @@ -157,7 +158,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 45, 1, + /* ColNames: */ 46, 1, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "cache_size", @@ -194,7 +195,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "collation_list", /* ePragTyp: */ PragTyp_COLLATION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 28, 2, + /* ColNames: */ 29, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) @@ -229,7 +230,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, - /* ColNames: */ 25, 3, + /* ColNames: */ 26, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) @@ -266,14 +267,14 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "foreign_key_check", /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 38, 4, + /* ColNames: */ 39, 4, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) {/* zName: */ "foreign_key_list", /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 30, 8, + /* ColNames: */ 31, 8, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -336,17 +337,17 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 11, 3, + /* ColNames: */ 12, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 20, 5, + /* ColNames: */ 21, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 14, 6, + /* ColNames: */ 15, 6, /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) @@ -393,7 +394,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "lock_status", /* ePragTyp: */ PragTyp_LOCK_STATUS, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 46, 2, + /* ColNames: */ 47, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -510,11 +511,11 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_SqlTrace }, #endif #endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 7, 4, + /* ColNames: */ 7, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -593,7 +594,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 42, 3, + /* ColNames: */ 43, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -604,4 +605,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 59 on by default, 73 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9a53d33626..6d58157b1b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1879,15 +1879,14 @@ struct Table { ** the TF_OOOHidden attribute would apply in this case. Such tables require ** special handling during INSERT processing. */ -#define TF_Readonly 0x01 /* Read-only system table */ -#define TF_Ephemeral 0x02 /* An ephemeral table */ -#define TF_HasPrimaryKey 0x04 /* Table has a primary key */ -#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ -/* available for reuse: 0x10 */ -#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ - +#define TF_Readonly 0x0001 /* Read-only system table */ +#define TF_Ephemeral 0x0002 /* An ephemeral table */ +#define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ +#define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ +#define TF_HasStat1 0x0010 /* Has STAT1 data */ +#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ /* ** Test to see whether or not a table is a virtual table. This is @@ -2130,6 +2129,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ + unsigned hasStat1:1; /* True if stat1 data has been loaded */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index c22f72d491..1c8bad54d3 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,8 +225,8 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height - IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + COLS: table index width height flags + IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info TYPE: INDEX_INFO From a3928dd7bee435f2cf4288b336968f6334376d39 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 15:26:36 +0000 Subject: [PATCH 02/12] Set the TF_StatsUsed flag on tables when the query planner outcome is affected by the sqlite_stat1 data. Also, change the column names of the "PRAGMA stats" command so that they are not keywords. FossilOrigin-Name: fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 --- manifest | 24 ++++---- manifest.uuid | 2 +- src/pragma.h | 10 ++-- src/sqliteInt.h | 7 ++- src/where.c | 13 +++++ src/whereInt.h | 5 ++ test/autoanalyze1.test | 124 +++++++++++++++++++++++++++++++++++++++++ tool/mkpragmatab.tcl | 2 +- 8 files changed, 165 insertions(+), 22 deletions(-) create mode 100644 test/autoanalyze1.test diff --git a/manifest b/manifest index 04051afae1..360ecf7903 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sIndex\sand\sTable\sobjects\sso\sthat\sthey\sremember\sif\stheir\sstats\scome\nfrom\sthe\ssqlite_stat1\stable.\s\sMake\sthe\s"PRAGMA\sstats"\san\sSQLITE_DEBUG\sonly\npragma.\s\sAdd\sthe\sflags\scolumn\sto\s"PRAGMA\sstats".\s\sThese\sare\sall\spreliminary\nsteps\stoward\sa\s"PRAGMA\sanalyze_ifneeded;"\sfeature. -D 2017-02-17T13:38:15.256 +C Set\sthe\sTF_StatsUsed\sflag\son\stables\swhen\sthe\squery\splanner\soutcome\sis\naffected\sby\sthe\ssqlite_stat1\sdata.\s\sAlso,\schange\sthe\scolumn\snames\sof\sthe\n"PRAGMA\sstats"\scommand\sso\sthat\sthey\sare\snot\skeywords. +D 2017-02-17T15:26:36.765 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,7 +388,7 @@ F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 -F src/pragma.h cea24a631982fd1a26fcddd46f596d22303b4247 +F src/pragma.h 9e65a9033cce7387c27cd89aecb2b2f205d1edf7 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 54bc20855f2a1a99a1b726f1384b5ce9ab67a0ff +F src/sqliteInt.h 5d50606deed2b38b35fb1b5e4ab658f8faa37d4a F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -475,8 +475,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 01baf58b72f3ddb7793cdee2871f751e3e09b35e -F src/whereInt.h c0b092180f04608d80c258174b0a14a1f9c8d02f +F src/where.c 1a3a8adb717a20f17c186f3baa22b0b5f3a5ab13 +F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -521,6 +521,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f +F test/autoanalyze1.test b31c18e73cf4e97b896288d8c817fce743f23e76 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1494,7 +1495,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 9c0a855e0daf83e54ffd9fd438260cd0c92f25d0 +F tool/mkpragmatab.tcl c955db934f7b1d800081447042cef692f26b2516 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1556,10 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 218b2bbb0de07288889f6762d4461ea8acd78969 -R af8aeffdf276070e8e8912906300a1a4 -T *branch * auto-analyze -T *sym-auto-analyze * -T -sym-trunk * +P 85026c8ee143bbd46565660fff8346ef81421546 +R 7cc0cc258beb93b198b99dc8e7b58520 U drh -Z f8720207819c88fb362c87de069d287a +Z 76eea0dfcfb8fa07347b6cdca98e2297 diff --git a/manifest.uuid b/manifest.uuid index fd9de00f55..1142241825 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85026c8ee143bbd46565660fff8346ef81421546 \ No newline at end of file +fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 \ No newline at end of file diff --git a/src/pragma.h b/src/pragma.h index 2a96ecaa43..1592bbd2d5 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -71,11 +71,11 @@ static const char *const pragCName[] = { /* 4 */ "notnull", /* 5 */ "dflt_value", /* 6 */ "pk", - /* 7 */ "table", /* Used by: stats */ - /* 8 */ "index", - /* 9 */ "width", - /* 10 */ "height", - /* 11 */ "flags", + /* 7 */ "tbl", /* Used by: stats */ + /* 8 */ "idx", + /* 9 */ "wdth", + /* 10 */ "hght", + /* 11 */ "flgs", /* 12 */ "seqno", /* Used by: index_info */ /* 13 */ "cid", /* 14 */ "name", diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6d58157b1b..3c75f2ff6e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1883,10 +1883,13 @@ struct Table { #define TF_Ephemeral 0x0002 /* An ephemeral table */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ -#define TF_HasStat1 0x0010 /* Has STAT1 data */ +#define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ #define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_SizeChng 0x0100 /* nRowLogEst might be inaccurate */ +#define TF_StatsUsed 0x0200 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ /* ** Test to see whether or not a table is a virtual table. This is @@ -2129,7 +2132,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ - unsigned hasStat1:1; /* True if stat1 data has been loaded */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/src/where.c b/src/where.c index f2bf400a2d..8d0dbb0979 100644 --- a/src/where.c +++ b/src/where.c @@ -2387,6 +2387,11 @@ static int whereLoopAddBtreeIndex( continue; } + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + }else{ + pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->u.btree.nBtm = saved_nBtm; @@ -2934,7 +2939,15 @@ static int whereLoopAddBtree( } } + pBuilder->bldFlags = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_StatsUsed; + } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; diff --git a/src/whereInt.h b/src/whereInt.h index c6195f53ed..f065fae6ba 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -395,8 +395,13 @@ struct WhereLoopBuilder { UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif + unsigned int bldFlags; /* SQLITE_BLDF_* flags */ }; +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test new file mode 100644 index 0000000000..c36e0fcbc7 --- /dev/null +++ b/test/autoanalyze1.test @@ -0,0 +1,124 @@ +# 2017-02-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for the logic used to estimate when +# running ANALYZE would be beneficial. +# +# Note that this test uses some hard-coded bitmask values from sqliteInt.h. +# If any of the following constants changes: +# +# define TF_HasStat1 0x0010 +# define TF_SizeChng 0x0100 +# define TF_StatsUsed 0x0200 +# +# then some of the magic numbers in test results below might need to be +# adjusted. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# There is nothing to test if ANALYZE is disable for this build. +# These tests also use "PRAGMA stats" which are only enabled for +# debugging builds. +# +ifcapable {!debug || !analyze} { + finish_test + return +} + +do_execsql_test autoanalyze1-100 { + -- Build up a test table with some indexes + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE UNIQUE INDEX t1bc ON t1(b,c); + CREATE INDEX t1d ON t1(d); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b,c,d) SELECT x, x, x, x FROM c; + -- Verify that the hasStat1 flag is clear on on indexes + SELECT idx, flgs FROM pragma_stats + WHERE idx IS NOT NULL + ORDER BY idx; + -- Verify that the TF_HasStat1 flag is clear on the table + SELECT tbl, (flgs & 0x10)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {t1bc 0 t1d 0 t1 0} + +# No use of stat1 recorded so far +do_execsql_test autoanalyze1-110 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Access using a unique index does not set the TF_StatsUsed flag. +# +do_execsql_test autoanalyze1-200 { + SELECT * FROM t1 WHERE a=55; +} {55 55 55 55} +do_execsql_test autoanalyze1-201 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-210 { + SELECT * FROM t1 WHERE a IN (55,199,299); +} {55 55 55 55} +do_execsql_test autoanalyze1-211 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-220 { + SELECT * FROM t1 WHERE (b,c)=(45,45); +} {45 45 45 45} +do_execsql_test autoanalyze1-221 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Any use of the non-unique t1d index triggers the use of stats. +# +sqlite3 db test.db +do_execsql_test autoanalyze1-300 { + SELECT * FROM t1 WHERE d=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-301 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +sqlite3 db test.db +do_execsql_test autoanalyze1-310 { + SELECT * FROM t1 WHERE d=45 AND a=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-311 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} ;# The ROWID lookup short-circuits the d=45 constraint + +sqlite3 db test.db +do_execsql_test autoanalyze1-320 { + SELECT * FROM t1 WHERE d=45 AND a IN (45,46); +} {45 45 45 45} +do_execsql_test autoanalyze1-321 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# Any use of prefix of a unique index triggers the use of stats +# +sqlite3 db test.db +do_execsql_test autoanalyze1-400 { + SELECT * FROM t1 WHERE b=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-401 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# The TF_StatsUsed flag is reset when the database is reopened +# +sqlite3 db test.db +do_execsql_test autoanalyze1-500 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +finish_test diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 1c8bad54d3..005c4bc48e 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,7 +225,7 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height flags + COLS: tbl idx wdth hght flgs IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info From 72052a73a3d1fc75015aa4123bea9af2f5a58e3f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 16:26:34 +0000 Subject: [PATCH 03/12] Add the "PRAGMA analyze_as_needed" command. FossilOrigin-Name: e93db2373127d31d33ec46ef918fa9386bb664a6 --- manifest | 22 +++++------ manifest.uuid | 2 +- src/analyze.c | 88 ++++++++++++++++++++++++++++++----------- src/parse.y | 4 +- src/pragma.c | 8 ++++ src/pragma.h | 94 +++++++++++++++++++++++--------------------- src/sqliteInt.h | 2 +- tool/mkpragmatab.tcl | 3 ++ 8 files changed, 141 insertions(+), 82 deletions(-) diff --git a/manifest b/manifest index 360ecf7903..9d27e603a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Set\sthe\sTF_StatsUsed\sflag\son\stables\swhen\sthe\squery\splanner\soutcome\sis\naffected\sby\sthe\ssqlite_stat1\sdata.\s\sAlso,\schange\sthe\scolumn\snames\sof\sthe\n"PRAGMA\sstats"\scommand\sso\sthat\sthey\sare\snot\skeywords. -D 2017-02-17T15:26:36.765 +C Add\sthe\s"PRAGMA\sanalyze_as_needed"\scommand. +D 2017-02-17T16:26:34.780 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 1b7197d619788353437d390de42a9bccbb4aa2ac +F src/analyze.c 39206f31ca136a0a4d4f1d6bfd00bc81c6b3e5ec F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -383,12 +383,12 @@ F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c +F src/parse.y d5695ae4887a52352b1b8a3e3623477c7e79abeb F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 -F src/pragma.h 9e65a9033cce7387c27cd89aecb2b2f205d1edf7 +F src/pragma.c e8b2ea66dfb1b90c53b920c08914ac26b109053a +F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 5d50606deed2b38b35fb1b5e4ab658f8faa37d4a +F src/sqliteInt.h 85706e0963964336270590a1e508dec78540bc5c F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1495,7 +1495,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl c955db934f7b1d800081447042cef692f26b2516 +F tool/mkpragmatab.tcl 6fd5ecb4a8debee39e3bb4e63f0634f69edfb861 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 85026c8ee143bbd46565660fff8346ef81421546 -R 7cc0cc258beb93b198b99dc8e7b58520 +P fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 +R 23ff1c8d68460fd739a12857d6287bf8 U drh -Z 76eea0dfcfb8fa07347b6cdca98e2297 +Z 4884503796ef3cb7f91c7100ffa75ea8 diff --git a/manifest.uuid b/manifest.uuid index 1142241825..f532ee84dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 \ No newline at end of file +e93db2373127d31d33ec46ef918fa9386bb664a6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 3ecf469cd3..34de7fc744 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1286,28 +1286,66 @@ static void loadAnalysis(Parse *pParse, int iDb){ } /* -** Generate code that will do an analysis of an entire database +** Return true if table pTab is in need of being reanalyzed. */ -static void analyzeDatabase(Parse *pParse, int iDb){ +static int analyzeNeeded(Table *pTab){ + Index *pIdx; + if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; + if( (pTab->tabFlags & TF_SizeChng)!=0 ) return 1; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ) return 1; + } + return 0; +} + +/* +** Generate code that will do an analysis of an entire database. +** +** Return a count of the number of tables actually analyzed. Return 0 +** if nothing was analyzed. +*/ +static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ sqlite3 *db = pParse->db; - Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + Schema *pSchema; HashElem *k; int iStatCur; int iMem; int iTab; + int iDb; /* Database currently being analyzed */ + int iDbFirst, iDbLast; /* Range of databases to be analyzed */ + int cnt; /* Number of tables analyzed in a single database */ + int allCnt = 0; /* Number of tables analyzed across all databases */ - sqlite3BeginWriteOperation(pParse, 0, iDb); - iStatCur = pParse->nTab; - pParse->nTab += 3; - openStatTable(pParse, iDb, iStatCur, 0, 0); - iMem = pParse->nMem+1; - iTab = pParse->nTab; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ - Table *pTab = (Table*)sqliteHashData(k); - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + if( iDbReq>=0 ){ + iDbFirst = iDbLast = iDbReq; + }else{ + iDbFirst = 0; + iDbLast = db->nDb-1; } - loadAnalysis(pParse, iDb); + for(iDb=iDbFirst; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; /* Do not analyze the TEMP database */ + pSchema = db->aDb[iDb].pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + cnt = 0; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + if( !onlyIfNeeded || analyzeNeeded(pTab) ){ + if( cnt==0 ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + } + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + cnt++; + allCnt++; + } + } + if( cnt ) loadAnalysis(pParse, iDb); + } + return allCnt; } /* @@ -1345,16 +1383,21 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ ** Form 1 causes all indices in all attached databases to be analyzed. ** Form 2 analyzes all indices the single database named. ** Form 3 analyzes all indices associated with the named table. +** +** If pName1 and pName2 are both NULL and if the ifNeeded flag is true, +** this routine computes an conditional ANALYZE on only those tables +** are believed to be in need of analysis. The conditional analysis +** might well be a no-op. */ -void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ +void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ sqlite3 *db = pParse->db; int iDb; - int i; char *z, *zDb; Table *pTab; Index *pIdx; Token *pTableName; Vdbe *v; + int needExpire = 1; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1366,15 +1409,12 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ assert( pName2!=0 || pName1==0 ); if( pName1==0 ){ /* Form 1: Analyze everything */ - for(i=0; inDb; i++){ - if( i==1 ) continue; /* Do not analyze the TEMP database */ - analyzeDatabase(pParse, i); - } + needExpire = analyzeDatabase(pParse, -1, ifNeeded) || ifNeeded==0; }else if( pName2->n==0 ){ /* Form 2: Analyze the database or table named */ iDb = sqlite3FindDb(db, pName1); if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); + analyzeDatabase(pParse, iDb, 0); }else{ z = sqlite3NameFromToken(db, pName1); if( z ){ @@ -1402,8 +1442,10 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); + if( needExpire ){ + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); + } } /* diff --git a/src/parse.y b/src/parse.y index 9cada2a1be..ff03c128ac 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1512,8 +1512,8 @@ cmd ::= REINDEX nm(X) dbnm(Y). {sqlite3Reindex(pParse, &X, &Y);} /////////////////////////////////// ANALYZE /////////////////////////////////// %ifndef SQLITE_OMIT_ANALYZE -cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0);} -cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} +cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0, 0);} +cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y, 0);} %endif //////////////////////// ALTER TABLE table ... //////////////////////////////// diff --git a/src/pragma.c b/src/pragma.c index f28ff9d13c..789751958c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1828,6 +1828,14 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA analyze_as_needed + */ + case PragTyp_ANALYZE_AS_NEEDED: { + sqlite3Analyze(pParse, 0, 0, 1); + break; + } + /* ** PRAGMA busy_timeout ** PRAGMA busy_timeout = N diff --git a/src/pragma.h b/src/pragma.h index 1592bbd2d5..ba107ec07e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -5,49 +5,50 @@ */ /* The various pragma types */ -#define PragTyp_HEADER_VALUE 0 -#define PragTyp_AUTO_VACUUM 1 -#define PragTyp_FLAG 2 -#define PragTyp_BUSY_TIMEOUT 3 -#define PragTyp_CACHE_SIZE 4 -#define PragTyp_CACHE_SPILL 5 -#define PragTyp_CASE_SENSITIVE_LIKE 6 -#define PragTyp_COLLATION_LIST 7 -#define PragTyp_COMPILE_OPTIONS 8 -#define PragTyp_DATA_STORE_DIRECTORY 9 -#define PragTyp_DATABASE_LIST 10 -#define PragTyp_DEFAULT_CACHE_SIZE 11 -#define PragTyp_ENCODING 12 -#define PragTyp_FOREIGN_KEY_CHECK 13 -#define PragTyp_FOREIGN_KEY_LIST 14 -#define PragTyp_INCREMENTAL_VACUUM 15 -#define PragTyp_INDEX_INFO 16 -#define PragTyp_INDEX_LIST 17 -#define PragTyp_INTEGRITY_CHECK 18 -#define PragTyp_JOURNAL_MODE 19 -#define PragTyp_JOURNAL_SIZE_LIMIT 20 -#define PragTyp_LOCK_PROXY_FILE 21 -#define PragTyp_LOCKING_MODE 22 -#define PragTyp_PAGE_COUNT 23 -#define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_SYNCHRONOUS 29 -#define PragTyp_TABLE_INFO 30 -#define PragTyp_TEMP_STORE 31 -#define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_THREADS 33 -#define PragTyp_WAL_AUTOCHECKPOINT 34 -#define PragTyp_WAL_CHECKPOINT 35 -#define PragTyp_ACTIVATE_EXTENSIONS 36 -#define PragTyp_HEXKEY 37 -#define PragTyp_KEY 38 -#define PragTyp_REKEY 39 -#define PragTyp_LOCK_STATUS 40 -#define PragTyp_PARSER_TRACE 41 -#define PragTyp_STATS 42 +#define PragTyp_ANALYZE_AS_NEEDED 0 +#define PragTyp_HEADER_VALUE 1 +#define PragTyp_AUTO_VACUUM 2 +#define PragTyp_FLAG 3 +#define PragTyp_BUSY_TIMEOUT 4 +#define PragTyp_CACHE_SIZE 5 +#define PragTyp_CACHE_SPILL 6 +#define PragTyp_CASE_SENSITIVE_LIKE 7 +#define PragTyp_COLLATION_LIST 8 +#define PragTyp_COMPILE_OPTIONS 9 +#define PragTyp_DATA_STORE_DIRECTORY 10 +#define PragTyp_DATABASE_LIST 11 +#define PragTyp_DEFAULT_CACHE_SIZE 12 +#define PragTyp_ENCODING 13 +#define PragTyp_FOREIGN_KEY_CHECK 14 +#define PragTyp_FOREIGN_KEY_LIST 15 +#define PragTyp_INCREMENTAL_VACUUM 16 +#define PragTyp_INDEX_INFO 17 +#define PragTyp_INDEX_LIST 18 +#define PragTyp_INTEGRITY_CHECK 19 +#define PragTyp_JOURNAL_MODE 20 +#define PragTyp_JOURNAL_SIZE_LIMIT 21 +#define PragTyp_LOCK_PROXY_FILE 22 +#define PragTyp_LOCKING_MODE 23 +#define PragTyp_PAGE_COUNT 24 +#define PragTyp_MMAP_SIZE 25 +#define PragTyp_PAGE_SIZE 26 +#define PragTyp_SECURE_DELETE 27 +#define PragTyp_SHRINK_MEMORY 28 +#define PragTyp_SOFT_HEAP_LIMIT 29 +#define PragTyp_SYNCHRONOUS 30 +#define PragTyp_TABLE_INFO 31 +#define PragTyp_TEMP_STORE 32 +#define PragTyp_TEMP_STORE_DIRECTORY 33 +#define PragTyp_THREADS 34 +#define PragTyp_WAL_AUTOCHECKPOINT 35 +#define PragTyp_WAL_CHECKPOINT 36 +#define PragTyp_ACTIVATE_EXTENSIONS 37 +#define PragTyp_HEXKEY 38 +#define PragTyp_KEY 39 +#define PragTyp_REKEY 40 +#define PragTyp_LOCK_STATUS 41 +#define PragTyp_PARSER_TRACE 42 +#define PragTyp_STATS 43 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -132,6 +133,11 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif + {/* zName: */ "analyze_as_needed", + /* ePragTyp: */ PragTyp_ANALYZE_AS_NEEDED, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -605,4 +611,4 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 73 total. */ +/* Number of pragmas: 60 on by default, 74 total. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3c75f2ff6e..f590c85f93 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4028,7 +4028,7 @@ void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*, u8*); -void sqlite3Analyze(Parse*, Token*, Token*); +void sqlite3Analyze(Parse*, Token*, Token*, int); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 005c4bc48e..2201ea1306 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -361,6 +361,9 @@ set pragma_def { NAME: threads FLAG: Result0 + + NAME: analyze_as_needed + FLAG: NoColumns } # Open the output file From 5e98e838da57d5d70c1b3321c73e82206f5c8457 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Feb 2017 19:24:06 +0000 Subject: [PATCH 04/12] The analyze_as_needed pragma now responds to table size growth and will automatically rerun the analysis after each 10x size increase. FossilOrigin-Name: bfbdd07409688fac4ccddbab3639745f6152e23d --- manifest | 22 ++++++++++----------- manifest.uuid | 2 +- src/analyze.c | 43 +++++++++++++++++++++++++++++++++--------- src/btree.c | 19 +++++++++++++++++++ src/btree.h | 1 + src/sqliteInt.h | 3 +-- src/vdbe.c | 27 ++++++++++++++++++++++++++ test/autoanalyze1.test | 21 ++++++++++----------- 8 files changed, 104 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 9d27e603a6..42a11a725c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"PRAGMA\sanalyze_as_needed"\scommand. -D 2017-02-17T16:26:34.780 +C The\sanalyze_as_needed\spragma\snow\sresponds\sto\stable\ssize\sgrowth\sand\swill\nautomatically\srerun\sthe\sanalysis\safter\seach\s10x\ssize\sincrease. +D 2017-02-17T19:24:06.618 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,14 +331,14 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c 39206f31ca136a0a4d4f1d6bfd00bc81c6b3e5ec +F src/analyze.c acef9c48eac30438e82f39eb8df6682c6e33942e F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 -F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac +F src/btree.c e7d724b02365d88920171caf0c919aa96d9b19e3 +F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 85706e0963964336270590a1e508dec78540bc5c +F src/sqliteInt.h 60c38b84a796cc9d0a9a794cca254b09898b9cc5 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -461,7 +461,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b +F src/vdbe.c ac8538b96af5c63d6f18484ffd981d4f44db0073 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -521,7 +521,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f -F test/autoanalyze1.test b31c18e73cf4e97b896288d8c817fce743f23e76 +F test/autoanalyze1.test 1ba80d5e1412d46503566b70741a5eea060da929 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 -R 23ff1c8d68460fd739a12857d6287bf8 +P e93db2373127d31d33ec46ef918fa9386bb664a6 +R 484e61dbf95be1098adb3acfa069e723 U drh -Z 4884503796ef3cb7f91c7100ffa75ea8 +Z 81466028927592a195c5763d3460fcde diff --git a/manifest.uuid b/manifest.uuid index f532ee84dd..8be7eeb5b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93db2373127d31d33ec46ef918fa9386bb664a6 \ No newline at end of file +bfbdd07409688fac4ccddbab3639745f6152e23d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 34de7fc744..082713fd23 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -958,10 +958,12 @@ static void analyzeOneTable( Index *pOnlyIdx, /* If not NULL, only analyze this one index */ int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ int iMem, /* Available memory locations begin here */ - int iTab /* Next available cursor */ + int iTab, /* Next available cursor */ + LogEst szOld /* Run the analysis if table row count is larger than this */ ){ sqlite3 *db = pParse->db; /* Database handle */ Index *pIdx; /* An index to being analyzed */ + int addrSizeCk = 0; /* Address of the IfSmaller */ int iIdxCur; /* Cursor open on index being analyzed */ int iTabCur; /* Table cursor */ Vdbe *v; /* The virtual machine being built up */ @@ -1014,6 +1016,9 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szOld>0 ){ + addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); + } sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ @@ -1271,6 +1276,7 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } + sqlite3VdbeJumpHere(v, addrSizeCk); } @@ -1286,16 +1292,33 @@ static void loadAnalysis(Parse *pParse, int iDb){ } /* -** Return true if table pTab is in need of being reanalyzed. +** Return true if table pTab might need to being reanalyzed. Return +** false if we know that pTab should not be reanalyzed. +** +** If returning true, also set *pThreshold to a size threshold that +** will determine at run-time whether or not the reanalysis occurs. +** The reanalysis will only occur if the size of the table is greater +** than the threshold. Not that the threshold is a logarithmic LogEst +** value. */ -static int analyzeNeeded(Table *pTab){ +static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ Index *pIdx; if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; - if( (pTab->tabFlags & TF_SizeChng)!=0 ) return 1; + + /* If TF_StatsUsed is true, then we might need to reanalyze. But + ** only reanalyze if the table size has grown by a factor of 10 or more */ + *pThreshold = pTab->nRowLogEst + 33; assert( sqlite3LogEst(10)==33 ); + + /* Except, if any of the indexes of the table do not have valid + ** sqlite_stat1 entries, then set the size threshold to zero to + ** ensure the analysis will always occur. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pIdx->hasStat1 ) return 1; + if( !pIdx->hasStat1 ){ + *pThreshold = 0; + break; + } } - return 0; + return 1; } /* @@ -1329,7 +1352,8 @@ static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ cnt = 0; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); - if( !onlyIfNeeded || analyzeNeeded(pTab) ){ + LogEst szThreshold = 0; + if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){ if( cnt==0 ){ sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; @@ -1338,7 +1362,7 @@ static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ iMem = pParse->nMem+1; iTab = pParse->nTab; } - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold); cnt++; allCnt++; } @@ -1368,7 +1392,8 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1, + pParse->nTab, 0); loadAnalysis(pParse, iDb); } diff --git a/src/btree.c b/src/btree.c index e78ffef1be..8aed08cb36 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5319,6 +5319,25 @@ int sqlite3BtreeEof(BtCursor *pCur){ return (CURSOR_VALID!=pCur->eState); } +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + if( pCur->eState!=CURSOR_VALID ) return -1; + if( pCur->apPage[pCur->iPage-1]->leaf==0 ) return -1; + for(n=1, i=0; iiPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor diff --git a/src/btree.h b/src/btree.h index ae57468e3f..0e017f5300 100644 --- a/src/btree.h +++ b/src/btree.h @@ -296,6 +296,7 @@ u32 sqlite3BtreePayloadSize(BtCursor*); char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); +i64 sqlite3BtreeRowCountEst(BtCursor*); #ifndef SQLITE_OMIT_INCRBLOB int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f590c85f93..d9cad11bcf 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1887,8 +1887,7 @@ struct Table { #define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ -#define TF_SizeChng 0x0100 /* nRowLogEst might be inaccurate */ -#define TF_StatsUsed 0x0200 /* Query planner decisions affected by +#define TF_StatsUsed 0x0100 /* Query planner decisions affected by ** Index.aiRowLogEst[] values */ /* diff --git a/src/vdbe.c b/src/vdbe.c index 7f286b2c6d..b595c4fb01 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4847,6 +4847,33 @@ case OP_Last: { /* jump */ break; } +/* Opcode: IfSmaller P1 P2 P3 * * +** +** Estimate the number of rows in the table P1. Jump to P2 if that +** estimate is less than approximately 2**(0.1*P3). +*/ +case OP_IfSmaller: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + i64 sz; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + sz = sqlite3BtreeRowCountEst(pCrsr); + if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + } + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: SorterSort P1 P2 * * * ** diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test index c36e0fcbc7..77e767a89e 100644 --- a/test/autoanalyze1.test +++ b/test/autoanalyze1.test @@ -16,8 +16,7 @@ # If any of the following constants changes: # # define TF_HasStat1 0x0010 -# define TF_SizeChng 0x0100 -# define TF_StatsUsed 0x0200 +# define TF_StatsUsed 0x0100 # # then some of the magic numbers in test results below might need to be # adjusted. @@ -52,7 +51,7 @@ do_execsql_test autoanalyze1-100 { # No use of stat1 recorded so far do_execsql_test autoanalyze1-110 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} # Access using a unique index does not set the TF_StatsUsed flag. @@ -61,21 +60,21 @@ do_execsql_test autoanalyze1-200 { SELECT * FROM t1 WHERE a=55; } {55 55 55 55} do_execsql_test autoanalyze1-201 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} do_execsql_test autoanalyze1-210 { SELECT * FROM t1 WHERE a IN (55,199,299); } {55 55 55 55} do_execsql_test autoanalyze1-211 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} do_execsql_test autoanalyze1-220 { SELECT * FROM t1 WHERE (b,c)=(45,45); } {45 45 45 45} do_execsql_test autoanalyze1-221 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} # Any use of the non-unique t1d index triggers the use of stats. @@ -85,7 +84,7 @@ do_execsql_test autoanalyze1-300 { SELECT * FROM t1 WHERE d=45; } {45 45 45 45} do_execsql_test autoanalyze1-301 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {1} sqlite3 db test.db @@ -93,7 +92,7 @@ do_execsql_test autoanalyze1-310 { SELECT * FROM t1 WHERE d=45 AND a=45; } {45 45 45 45} do_execsql_test autoanalyze1-311 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} ;# The ROWID lookup short-circuits the d=45 constraint sqlite3 db test.db @@ -101,7 +100,7 @@ do_execsql_test autoanalyze1-320 { SELECT * FROM t1 WHERE d=45 AND a IN (45,46); } {45 45 45 45} do_execsql_test autoanalyze1-321 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {1} # Any use of prefix of a unique index triggers the use of stats @@ -111,14 +110,14 @@ do_execsql_test autoanalyze1-400 { SELECT * FROM t1 WHERE b=45; } {45 45 45 45} do_execsql_test autoanalyze1-401 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {1} # The TF_StatsUsed flag is reset when the database is reopened # sqlite3 db test.db do_execsql_test autoanalyze1-500 { - SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; + SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; } {0} finish_test From 182e84c116eb9b6f3e9dc3ee1a694e302cf6c08d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 02:19:02 +0000 Subject: [PATCH 05/12] In the analyze_as_needed pragma, avoid running unnecessary OP_LoadAnalysis and OP_Expire opcodes. Make the analyze_as_needed pragma responsive to the schema name. FossilOrigin-Name: 882599a4a7ea92c9e7752e0745475508e58a11c3 --- manifest | 18 +++++----- manifest.uuid | 2 +- src/analyze.c | 90 ++++++++++++++++++++++++++----------------------- src/parse.y | 4 +-- src/pragma.c | 2 +- src/sqliteInt.h | 3 +- 6 files changed, 63 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index 42a11a725c..d311cdbbf3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sanalyze_as_needed\spragma\snow\sresponds\sto\stable\ssize\sgrowth\sand\swill\nautomatically\srerun\sthe\sanalysis\safter\seach\s10x\ssize\sincrease. -D 2017-02-17T19:24:06.618 +C In\sthe\sanalyze_as_needed\spragma,\savoid\srunning\sunnecessary\sOP_LoadAnalysis\s\nand\sOP_Expire\sopcodes.\s\sMake\sthe\sanalyze_as_needed\spragma\sresponsive\sto\sthe\nschema\sname. +D 2017-02-18T02:19:02.183 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c acef9c48eac30438e82f39eb8df6682c6e33942e +F src/analyze.c eb50045b8f2e0d8a0a36a2158a65afe098d9a3bb F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -383,11 +383,11 @@ F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa -F src/parse.y d5695ae4887a52352b1b8a3e3623477c7e79abeb +F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c e8b2ea66dfb1b90c53b920c08914ac26b109053a +F src/pragma.c cf0f101d2986d258699cbbd249392858ec9a6b7e F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 60c38b84a796cc9d0a9a794cca254b09898b9cc5 +F src/sqliteInt.h 70abfa92e02688f1d189bf63fd25d309ca8d8d1e F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e93db2373127d31d33ec46ef918fa9386bb664a6 -R 484e61dbf95be1098adb3acfa069e723 +P bfbdd07409688fac4ccddbab3639745f6152e23d +R 1c1b7451b307830f90322e6d51007af7 U drh -Z 81466028927592a195c5763d3460fcde +Z 2d7c806c83a06bb426974433c00a5916 diff --git a/manifest.uuid b/manifest.uuid index 8be7eeb5b9..9825130b6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bfbdd07409688fac4ccddbab3639745f6152e23d \ No newline at end of file +882599a4a7ea92c9e7752e0745475508e58a11c3 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 082713fd23..c30027aec9 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -971,6 +971,7 @@ static void analyzeOneTable( int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ + int regWorkDone = iMem++; /* Set to 1 if any work is done */ int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regChng = iMem++; /* Index of changed index field */ @@ -1018,7 +1019,7 @@ static void analyzeOneTable( sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); if( szOld>0 ){ addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); - } + } sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ @@ -1276,21 +1277,11 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } + sqlite3VdbeAddOp2(v, OP_Integer, 1, regWorkDone); + VdbeComment((v, "work was done")); sqlite3VdbeJumpHere(v, addrSizeCk); } - -/* -** Generate code that will cause the most recent index analysis to -** be loaded into internal hash tables where is can be used. -*/ -static void loadAnalysis(Parse *pParse, int iDb){ - Vdbe *v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); - } -} - /* ** Return true if table pTab might need to being reanalyzed. Return ** false if we know that pTab should not be reanalyzed. @@ -1322,54 +1313,70 @@ static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ } /* -** Generate code that will do an analysis of an entire database. +** Generate code that will do an analysis of an entire database, or +** all databases in the connection if iDbReq is negative. ** -** Return a count of the number of tables actually analyzed. Return 0 -** if nothing was analyzed. +** If onlyIfNeeded is true, then only run the analysis if SQLite thinks +** it is actually needed. */ -static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){ +void sqlite3AnalyzeDatabase( + Parse *pParse, /* The parsing context */ + int iDbReq, /* Which schema to analyze. -1 for all (except TEMP) */ + int onlyIfNeeded /* Only do the analysis if needed, when true */ +){ sqlite3 *db = pParse->db; Schema *pSchema; HashElem *k; - int iStatCur; - int iMem; - int iTab; + int iStatCur = 0; + int iMem = 0; + int iTab = 0; int iDb; /* Database currently being analyzed */ int iDbFirst, iDbLast; /* Range of databases to be analyzed */ - int cnt; /* Number of tables analyzed in a single database */ - int allCnt = 0; /* Number of tables analyzed across all databases */ + int bStatTabs = 0; + Vdbe *v = sqlite3GetVdbe(pParse); + int nHit = 0; + unsigned char aHit[SQLITE_MAX_ATTACHED+2]; + if( v==0 ) return; if( iDbReq>=0 ){ iDbFirst = iDbLast = iDbReq; }else{ iDbFirst = 0; iDbLast = db->nDb-1; } - for(iDb=iDbFirst; iDb<=iDbLast; iDb++){ + for(iDb=iDbFirst; iDb<=iDbLast; iDb++, bStatTabs=0){ if( iDb==1 ) continue; /* Do not analyze the TEMP database */ pSchema = db->aDb[iDb].pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - cnt = 0; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); LogEst szThreshold = 0; if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){ - if( cnt==0 ){ - sqlite3BeginWriteOperation(pParse, 0, iDb); + if( iMem==0 ){ iStatCur = pParse->nTab; pParse->nTab += 3; - openStatTable(pParse, iDb, iStatCur, 0, 0); iMem = pParse->nMem+1; iTab = pParse->nTab; + sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem); + } + if( !bStatTabs ){ + aHit[nHit++] = iDb; + sqlite3BeginWriteOperation(pParse, 0, iDb); + openStatTable(pParse, iDb, iStatCur, 0, 0); + bStatTabs = 1; } analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold); - cnt++; - allCnt++; } } - if( cnt ) loadAnalysis(pParse, iDb); } - return allCnt; + if( iMem ){ + int addrTop = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); VdbeCoverage(v); + for(iDb=0; iDbdb) ); @@ -1392,9 +1400,13 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1, + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1, pParse->nTab, 0); - loadAnalysis(pParse, iDb); + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + sqlite3VdbeAddOp0(v, OP_Expire); + } } /* @@ -1414,15 +1426,13 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ ** are believed to be in need of analysis. The conditional analysis ** might well be a no-op. */ -void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ +void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ sqlite3 *db = pParse->db; int iDb; char *z, *zDb; Table *pTab; Index *pIdx; Token *pTableName; - Vdbe *v; - int needExpire = 1; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1434,12 +1444,12 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ assert( pName2!=0 || pName1==0 ); if( pName1==0 ){ /* Form 1: Analyze everything */ - needExpire = analyzeDatabase(pParse, -1, ifNeeded) || ifNeeded==0; + sqlite3AnalyzeDatabase(pParse, -1, 0); }else if( pName2->n==0 ){ /* Form 2: Analyze the database or table named */ iDb = sqlite3FindDb(db, pName1); if( iDb>=0 ){ - analyzeDatabase(pParse, iDb, 0); + sqlite3AnalyzeDatabase(pParse, iDb, 0); }else{ z = sqlite3NameFromToken(db, pName1); if( z ){ @@ -1467,10 +1477,6 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2, int ifNeeded){ } } } - if( needExpire ){ - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); - } } /* diff --git a/src/parse.y b/src/parse.y index ff03c128ac..9cada2a1be 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1512,8 +1512,8 @@ cmd ::= REINDEX nm(X) dbnm(Y). {sqlite3Reindex(pParse, &X, &Y);} /////////////////////////////////// ANALYZE /////////////////////////////////// %ifndef SQLITE_OMIT_ANALYZE -cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0, 0);} -cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y, 0);} +cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0);} +cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} %endif //////////////////////// ALTER TABLE table ... //////////////////////////////// diff --git a/src/pragma.c b/src/pragma.c index 789751958c..3bcf62f69f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1832,7 +1832,7 @@ void sqlite3Pragma( ** PRAGMA analyze_as_needed */ case PragTyp_ANALYZE_AS_NEEDED: { - sqlite3Analyze(pParse, 0, 0, 1); + sqlite3AnalyzeDatabase(pParse, zDb ? iDb : -1, 1); break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d9cad11bcf..218c2c6af6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4027,7 +4027,8 @@ void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*, u8*); -void sqlite3Analyze(Parse*, Token*, Token*, int); +void sqlite3Analyze(Parse*, Token*, Token*); +void sqlite3AnalyzeDatabase(Parse*,int,int); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); From dfe11bae9944fa05066314dd9063ff834fd60dae Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 02:42:54 +0000 Subject: [PATCH 06/12] Fix errors in the table resize detection. FossilOrigin-Name: 4229caec0b60a1617b9d5ff94b47271cbd7be1e0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 8 +++++--- src/btree.c | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d311cdbbf3..5af18e20b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sanalyze_as_needed\spragma,\savoid\srunning\sunnecessary\sOP_LoadAnalysis\s\nand\sOP_Expire\sopcodes.\s\sMake\sthe\sanalyze_as_needed\spragma\sresponsive\sto\sthe\nschema\sname. -D 2017-02-18T02:19:02.183 +C Fix\serrors\sin\sthe\stable\sresize\sdetection. +D 2017-02-18T02:42:54.892 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,13 +331,13 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c eb50045b8f2e0d8a0a36a2158a65afe098d9a3bb +F src/analyze.c e01e5362ba2ac8430ab7833022afb6081ade8315 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c e7d724b02365d88920171caf0c919aa96d9b19e3 +F src/btree.c a4ab1fb5cdeea88c4f76216e41cfecfa505c8c43 F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bfbdd07409688fac4ccddbab3639745f6152e23d -R 1c1b7451b307830f90322e6d51007af7 +P 882599a4a7ea92c9e7752e0745475508e58a11c3 +R cf9f80fd407d6e9e7cfac9c2957017b5 U drh -Z 2d7c806c83a06bb426974433c00a5916 +Z 225df65ba1e601aba9cbc975df68b3a8 diff --git a/manifest.uuid b/manifest.uuid index 9825130b6a..07d49087a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -882599a4a7ea92c9e7752e0745475508e58a11c3 \ No newline at end of file +4229caec0b60a1617b9d5ff94b47271cbd7be1e0 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c30027aec9..118cdb9383 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1019,6 +1019,7 @@ static void analyzeOneTable( sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); if( szOld>0 ){ addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); + VdbeCoverage(v); } sqlite3VdbeLoadString(v, regTabname, pTab->zName); @@ -1296,9 +1297,10 @@ static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ Index *pIdx; if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; - /* If TF_StatsUsed is true, then we might need to reanalyze. But - ** only reanalyze if the table size has grown by a factor of 10 or more */ - *pThreshold = pTab->nRowLogEst + 33; assert( sqlite3LogEst(10)==33 ); + /* If TF_StatsUsed is true, then we might need to reanalyze. + ** TUNING: Only reanalyze if the table size has grown by a factor + ** of 25 or more. */ + *pThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); /* Except, if any of the indexes of the table do not have valid ** sqlite_stat1 entries, then set the size threshold to zero to diff --git a/src/btree.c b/src/btree.c index 8aed08cb36..4d31661f10 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5331,8 +5331,8 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); if( pCur->eState!=CURSOR_VALID ) return -1; - if( pCur->apPage[pCur->iPage-1]->leaf==0 ) return -1; - for(n=1, i=0; iiPage; i++){ + if( pCur->apPage[pCur->iPage]->leaf==0 ) return -1; + for(n=1, i=0; i<=pCur->iPage; i++){ n *= pCur->apPage[i]->nCell; } return n; From 4a54bb579480352dce25d2982b2f266c58255248 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 15:58:52 +0000 Subject: [PATCH 07/12] Add the OP_SqlExec opcode and use it to implement "PRAGMA analyze_as_needed", invoking ANALYZE subcommands as necessary. This simplifies the implementation. FossilOrigin-Name: d386015f5e7ecdd951d70db56b7bbd858be7ad90 --- manifest | 18 +++--- manifest.uuid | 2 +- src/analyze.c | 157 +++++++++++++----------------------------------- src/pragma.c | 36 ++++++++++- src/sqliteInt.h | 1 - src/vdbe.c | 10 +++ 6 files changed, 96 insertions(+), 128 deletions(-) diff --git a/manifest b/manifest index 5af18e20b2..984bdbb4fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\serrors\sin\sthe\stable\sresize\sdetection. -D 2017-02-18T02:42:54.892 +C Add\sthe\sOP_SqlExec\sopcode\sand\suse\sit\sto\simplement\s"PRAGMA\sanalyze_as_needed",\ninvoking\sANALYZE\ssubcommands\sas\snecessary.\s\sThis\ssimplifies\sthe\simplementation. +D 2017-02-18T15:58:52.451 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -331,7 +331,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c e01e5362ba2ac8430ab7833022afb6081ade8315 +F src/analyze.c 1b7197d619788353437d390de42a9bccbb4aa2ac F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -387,7 +387,7 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c cf0f101d2986d258699cbbd249392858ec9a6b7e +F src/pragma.c 1ed159f6fed5af7f5b095f55bdefcb0848956397 F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 70abfa92e02688f1d189bf63fd25d309ca8d8d1e +F src/sqliteInt.h 87857c2b8d8825e22970ce3f466b8618ee55b44e F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -461,7 +461,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 -F src/vdbe.c ac8538b96af5c63d6f18484ffd981d4f44db0073 +F src/vdbe.c 02f9db522c73ba4e1743585e9fd6f8ac6c71515b F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 882599a4a7ea92c9e7752e0745475508e58a11c3 -R cf9f80fd407d6e9e7cfac9c2957017b5 +P 4229caec0b60a1617b9d5ff94b47271cbd7be1e0 +R 03be76497e56750b0101f816e3587990 U drh -Z 225df65ba1e601aba9cbc975df68b3a8 +Z 8d178351b984357dd4d18d50fe476798 diff --git a/manifest.uuid b/manifest.uuid index 07d49087a1..323aca9202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4229caec0b60a1617b9d5ff94b47271cbd7be1e0 \ No newline at end of file +d386015f5e7ecdd951d70db56b7bbd858be7ad90 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 118cdb9383..3ecf469cd3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -958,12 +958,10 @@ static void analyzeOneTable( Index *pOnlyIdx, /* If not NULL, only analyze this one index */ int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ int iMem, /* Available memory locations begin here */ - int iTab, /* Next available cursor */ - LogEst szOld /* Run the analysis if table row count is larger than this */ + int iTab /* Next available cursor */ ){ sqlite3 *db = pParse->db; /* Database handle */ Index *pIdx; /* An index to being analyzed */ - int addrSizeCk = 0; /* Address of the IfSmaller */ int iIdxCur; /* Cursor open on index being analyzed */ int iTabCur; /* Table cursor */ Vdbe *v; /* The virtual machine being built up */ @@ -971,7 +969,6 @@ static void analyzeOneTable( int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ - int regWorkDone = iMem++; /* Set to 1 if any work is done */ int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regChng = iMem++; /* Index of changed index field */ @@ -1017,10 +1014,6 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - if( szOld>0 ){ - addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld); - VdbeCoverage(v); - } sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ @@ -1278,107 +1271,43 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regWorkDone); - VdbeComment((v, "work was done")); - sqlite3VdbeJumpHere(v, addrSizeCk); } -/* -** Return true if table pTab might need to being reanalyzed. Return -** false if we know that pTab should not be reanalyzed. -** -** If returning true, also set *pThreshold to a size threshold that -** will determine at run-time whether or not the reanalysis occurs. -** The reanalysis will only occur if the size of the table is greater -** than the threshold. Not that the threshold is a logarithmic LogEst -** value. -*/ -static int analyzeNeeded(Table *pTab, LogEst *pThreshold){ - Index *pIdx; - if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0; - - /* If TF_StatsUsed is true, then we might need to reanalyze. - ** TUNING: Only reanalyze if the table size has grown by a factor - ** of 25 or more. */ - *pThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); - - /* Except, if any of the indexes of the table do not have valid - ** sqlite_stat1 entries, then set the size threshold to zero to - ** ensure the analysis will always occur. */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pIdx->hasStat1 ){ - *pThreshold = 0; - break; - } - } - return 1; -} /* -** Generate code that will do an analysis of an entire database, or -** all databases in the connection if iDbReq is negative. -** -** If onlyIfNeeded is true, then only run the analysis if SQLite thinks -** it is actually needed. +** Generate code that will cause the most recent index analysis to +** be loaded into internal hash tables where is can be used. */ -void sqlite3AnalyzeDatabase( - Parse *pParse, /* The parsing context */ - int iDbReq, /* Which schema to analyze. -1 for all (except TEMP) */ - int onlyIfNeeded /* Only do the analysis if needed, when true */ -){ - sqlite3 *db = pParse->db; - Schema *pSchema; - HashElem *k; - int iStatCur = 0; - int iMem = 0; - int iTab = 0; - int iDb; /* Database currently being analyzed */ - int iDbFirst, iDbLast; /* Range of databases to be analyzed */ - int bStatTabs = 0; +static void loadAnalysis(Parse *pParse, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); - int nHit = 0; - unsigned char aHit[SQLITE_MAX_ATTACHED+2]; + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + } +} - if( v==0 ) return; - if( iDbReq>=0 ){ - iDbFirst = iDbLast = iDbReq; - }else{ - iDbFirst = 0; - iDbLast = db->nDb-1; - } - for(iDb=iDbFirst; iDb<=iDbLast; iDb++, bStatTabs=0){ - if( iDb==1 ) continue; /* Do not analyze the TEMP database */ - pSchema = db->aDb[iDb].pSchema; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ - Table *pTab = (Table*)sqliteHashData(k); - LogEst szThreshold = 0; - if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){ - if( iMem==0 ){ - iStatCur = pParse->nTab; - pParse->nTab += 3; - iMem = pParse->nMem+1; - iTab = pParse->nTab; - sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem); - } - if( !bStatTabs ){ - aHit[nHit++] = iDb; - sqlite3BeginWriteOperation(pParse, 0, iDb); - openStatTable(pParse, iDb, iStatCur, 0, 0); - bStatTabs = 1; - } - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold); - } - } - } - if( iMem ){ - int addrTop = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); VdbeCoverage(v); - for(iDb=0; iDbdb; + Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + HashElem *k; + int iStatCur; + int iMem; + int iTab; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); } + loadAnalysis(pParse, iDb); } /* @@ -1389,7 +1318,6 @@ void sqlite3AnalyzeDatabase( static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ int iDb; int iStatCur; - Vdbe *v; assert( pTab!=0 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); @@ -1402,13 +1330,8 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1, - pParse->nTab, 0); - v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); - sqlite3VdbeAddOp0(v, OP_Expire); - } + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + loadAnalysis(pParse, iDb); } /* @@ -1422,19 +1345,16 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ ** Form 1 causes all indices in all attached databases to be analyzed. ** Form 2 analyzes all indices the single database named. ** Form 3 analyzes all indices associated with the named table. -** -** If pName1 and pName2 are both NULL and if the ifNeeded flag is true, -** this routine computes an conditional ANALYZE on only those tables -** are believed to be in need of analysis. The conditional analysis -** might well be a no-op. */ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ sqlite3 *db = pParse->db; int iDb; + int i; char *z, *zDb; Table *pTab; Index *pIdx; Token *pTableName; + Vdbe *v; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1446,12 +1366,15 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ assert( pName2!=0 || pName1==0 ); if( pName1==0 ){ /* Form 1: Analyze everything */ - sqlite3AnalyzeDatabase(pParse, -1, 0); + for(i=0; inDb; i++){ + if( i==1 ) continue; /* Do not analyze the TEMP database */ + analyzeDatabase(pParse, i); + } }else if( pName2->n==0 ){ /* Form 2: Analyze the database or table named */ iDb = sqlite3FindDb(db, pName1); if( iDb>=0 ){ - sqlite3AnalyzeDatabase(pParse, iDb, 0); + analyzeDatabase(pParse, iDb); }else{ z = sqlite3NameFromToken(db, pName1); if( z ){ @@ -1479,6 +1402,8 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* diff --git a/src/pragma.c b/src/pragma.c index 3bcf62f69f..49d711ded7 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1832,7 +1832,41 @@ void sqlite3Pragma( ** PRAGMA analyze_as_needed */ case PragTyp_ANALYZE_AS_NEEDED: { - sqlite3AnalyzeDatabase(pParse, zDb ? iDb : -1, 1); + int iDbLast; + int iTabCur; + HashElem *k; + Schema *pSchema; + Table *pTab; + Index *pIdx; + LogEst szThreshold; + char *zSubSql; + + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2, szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 218c2c6af6..19c7481223 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4028,7 +4028,6 @@ void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); char sqlite3AffinityType(const char*, u8*); void sqlite3Analyze(Parse*, Token*, Token*); -void sqlite3AnalyzeDatabase(Parse*,int,int); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); diff --git a/src/vdbe.c b/src/vdbe.c index b595c4fb01..b8050e8d50 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5518,6 +5518,16 @@ case OP_CreateTable: { /* out2 */ break; } +/* Opcode: SqlExec * * * P4 * +** +** Run the SQL statement or statements specified in the P4 string. +*/ +case OP_SqlExec: { + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + if( rc ) goto abort_due_to_error; + break; +} + /* Opcode: ParseSchema P1 * * P4 * ** ** Read and parse all entries from the SQLITE_MASTER table of database P1 From 99d5b4cde6c64a514d9531494bb6875bf5b2a46c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Feb 2017 22:52:40 +0000 Subject: [PATCH 08/12] Updated comments. No code changes. FossilOrigin-Name: e842ad391e62df273a5b1ed569d42ea46d03a99b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 41 ++++++++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 984bdbb4fc..420fc4058d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sOP_SqlExec\sopcode\sand\suse\sit\sto\simplement\s"PRAGMA\sanalyze_as_needed",\ninvoking\sANALYZE\ssubcommands\sas\snecessary.\s\sThis\ssimplifies\sthe\simplementation. -D 2017-02-18T15:58:52.451 +C Updated\scomments.\s\sNo\scode\schanges. +D 2017-02-18T22:52:40.654 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -387,7 +387,7 @@ F src/parse.y 591704fce84f814d9a3642774c1f011d38f4149c F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 1ed159f6fed5af7f5b095f55bdefcb0848956397 +F src/pragma.c 8a35509e8f60746fbdcde5312ddfd177c46c7e9c F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4229caec0b60a1617b9d5ff94b47271cbd7be1e0 -R 03be76497e56750b0101f816e3587990 +P d386015f5e7ecdd951d70db56b7bbd858be7ad90 +R bb5056af5425096e2e052e924ae83b2e U drh -Z 8d178351b984357dd4d18d50fe476798 +Z a573182d78be17d037798d5840f110fc diff --git a/manifest.uuid b/manifest.uuid index 323aca9202..d08504dc8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d386015f5e7ecdd951d70db56b7bbd858be7ad90 \ No newline at end of file +e842ad391e62df273a5b1ed569d42ea46d03a99b \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 49d711ded7..3aa07e8d18 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1830,16 +1830,33 @@ void sqlite3Pragma( /* ** PRAGMA analyze_as_needed + ** PRAGMA schema.analyze_as_needed + ** + ** This pragma runs ANALYZE on any tables which would have benefitted + ** from having recent statistics at some point since the start of the + ** current connection. Only tables in "schema" are analyzed in the + ** second form. In the first form, all tables except TEMP tables are + ** checked. + ** + ** A table is analyzed only if both of the following are true: + ** + ** (1) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (2) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. */ case PragTyp_ANALYZE_AS_NEEDED: { - int iDbLast; - int iTabCur; - HashElem *k; - Schema *pSchema; - Table *pTab; - Index *pIdx; - LogEst szThreshold; - char *zSubSql; + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ iTabCur = pParse->nTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ @@ -1848,11 +1865,17 @@ void sqlite3Pragma( pSchema = db->aDb[iDb].pSchema; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !pIdx->hasStat1 ){ - szThreshold = 0; + szThreshold = 0; /* Always analyze if any index lacks statistics */ break; } } From 2ead47cb49d28ead0ea67825bc66a5c360867201 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2017 20:24:10 +0000 Subject: [PATCH 09/12] Change the name of the analyze_as_needed pragma to "optimize". Enhance the comment (which will become documentation, assuming these changes land on trunk) to explain that the optimize pragma is likely to be enhanced in various ways in future releases and that applications should not depend upon the current behavior. FossilOrigin-Name: 9fced545a6f80c55d6dc4a6106cb2d3569566b3e --- manifest | 16 +++++------ manifest.uuid | 2 +- src/pragma.c | 30 ++++++++++++++++----- src/pragma.h | 64 +++++++++++++++++++++++--------------------- tool/mkpragmatab.tcl | 2 +- 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 392b3afc29..508790c7bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sintegrity_check\sand\sother\simprovements\sfrom\strunk. -D 2017-02-22T19:49:54.795 +C Change\sthe\sname\sof\sthe\sanalyze_as_needed\spragma\sto\s"optimize".\s\sEnhance\sthe\ncomment\s(which\swill\sbecome\sdocumentation,\sassuming\sthese\schanges\sland\son\strunk)\nto\sexplain\sthat\sthe\soptimize\spragma\sis\slikely\sto\sbe\senhanced\sin\svarious\sways\nin\sfuture\sreleases\sand\sthat\sapplications\sshould\snot\sdepend\supon\sthe\scurrent\nbehavior. +D 2017-02-22T20:24:10.705 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,8 +388,8 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 97742aae32e645926f6f8fc426169fb5ef97c268 -F src/pragma.h 065e184494f12e94111da1ab6984faa7b6142e68 +F src/pragma.c 46202f2f5ee6957ff5cba581ee3c36507685def0 +F src/pragma.h d97dd835c7f4dfb6857487707313385d44fa76c0 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -1496,7 +1496,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 6fd5ecb4a8debee39e3bb4e63f0634f69edfb861 +F tool/mkpragmatab.tcl 9b499f7301fadeddeae52b95f962ef7e5a718f50 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1558,7 +1558,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff213f2ef5bf96754a2264685d25546d8b5ccf0a d6afd98de3ee8b714dfd6477ead955096f623972 -R c12096fa1c0a2878a437d029e47aff9d +P fe073905081b421405ca425ca03c5b8b0ff5f2c8 +R b1ce2ff126387ffc06c8537ddbfa9731 U drh -Z 3c7beae38049bf1fe67e75122f4d0189 +Z 5165d34a54df32aa272e95c6a5e5953c diff --git a/manifest.uuid b/manifest.uuid index 54b7ffb813..8e0845a1c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe073905081b421405ca425ca03c5b8b0ff5f2c8 \ No newline at end of file +9fced545a6f80c55d6dc4a6106cb2d3569566b3e \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 7ff76d02d1..714fb413b6 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1863,16 +1863,29 @@ void sqlite3Pragma( } /* - ** PRAGMA analyze_as_needed - ** PRAGMA schema.analyze_as_needed + ** PRAGMA optimize + ** PRAGMA schema.optimize ** - ** This pragma runs ANALYZE on any tables which would have benefitted - ** from having recent statistics at some point since the start of the - ** current connection. Only tables in "schema" are analyzed in the + ** Attempt to optimize the database. All schemas are optimized in the first + ** form, and only the specified schema is optimized in the second form. + ** + ** The details of optimizations performed by this pragma does are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** Argments to this pragma are currently ignored, but future enhancements + ** might make use of arguments to control which optimizations are allowed + ** or to suggest limits on how much CPU time and I/O should be expended + ** in the optimization effort. + ** + ** The current implementation runs ANALYZE on any tables which might have + ** benefitted from having recent statistics at some point since the start + ** of the current connection. Only tables in "schema" are analyzed in the ** second form. In the first form, all tables except TEMP tables are ** checked. ** - ** A table is analyzed only if both of the following are true: + ** In the current implementation, a table is analyzed only if both of + ** the following are true: ** ** (1) The query planner used sqlite_stat1-style statistics for one or ** more indexes of the table at some point during the lifetime of @@ -1881,8 +1894,11 @@ void sqlite3Pragma( ** (2) One or more indexes of the table are currently unanalyzed OR ** the number of rows in the table has increased by 25 times or more ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. */ - case PragTyp_ANALYZE_AS_NEEDED: { + case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ int iTabCur; /* Cursor for a table whose size needs checking */ HashElem *k; /* Loop over tables of a schema */ diff --git a/src/pragma.h b/src/pragma.h index ba107ec07e..363975df55 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -5,32 +5,32 @@ */ /* The various pragma types */ -#define PragTyp_ANALYZE_AS_NEEDED 0 -#define PragTyp_HEADER_VALUE 1 -#define PragTyp_AUTO_VACUUM 2 -#define PragTyp_FLAG 3 -#define PragTyp_BUSY_TIMEOUT 4 -#define PragTyp_CACHE_SIZE 5 -#define PragTyp_CACHE_SPILL 6 -#define PragTyp_CASE_SENSITIVE_LIKE 7 -#define PragTyp_COLLATION_LIST 8 -#define PragTyp_COMPILE_OPTIONS 9 -#define PragTyp_DATA_STORE_DIRECTORY 10 -#define PragTyp_DATABASE_LIST 11 -#define PragTyp_DEFAULT_CACHE_SIZE 12 -#define PragTyp_ENCODING 13 -#define PragTyp_FOREIGN_KEY_CHECK 14 -#define PragTyp_FOREIGN_KEY_LIST 15 -#define PragTyp_INCREMENTAL_VACUUM 16 -#define PragTyp_INDEX_INFO 17 -#define PragTyp_INDEX_LIST 18 -#define PragTyp_INTEGRITY_CHECK 19 -#define PragTyp_JOURNAL_MODE 20 -#define PragTyp_JOURNAL_SIZE_LIMIT 21 -#define PragTyp_LOCK_PROXY_FILE 22 -#define PragTyp_LOCKING_MODE 23 -#define PragTyp_PAGE_COUNT 24 -#define PragTyp_MMAP_SIZE 25 +#define PragTyp_HEADER_VALUE 0 +#define PragTyp_AUTO_VACUUM 1 +#define PragTyp_FLAG 2 +#define PragTyp_BUSY_TIMEOUT 3 +#define PragTyp_CACHE_SIZE 4 +#define PragTyp_CACHE_SPILL 5 +#define PragTyp_CASE_SENSITIVE_LIKE 6 +#define PragTyp_COLLATION_LIST 7 +#define PragTyp_COMPILE_OPTIONS 8 +#define PragTyp_DATA_STORE_DIRECTORY 9 +#define PragTyp_DATABASE_LIST 10 +#define PragTyp_DEFAULT_CACHE_SIZE 11 +#define PragTyp_ENCODING 12 +#define PragTyp_FOREIGN_KEY_CHECK 13 +#define PragTyp_FOREIGN_KEY_LIST 14 +#define PragTyp_INCREMENTAL_VACUUM 15 +#define PragTyp_INDEX_INFO 16 +#define PragTyp_INDEX_LIST 17 +#define PragTyp_INTEGRITY_CHECK 18 +#define PragTyp_JOURNAL_MODE 19 +#define PragTyp_JOURNAL_SIZE_LIMIT 20 +#define PragTyp_LOCK_PROXY_FILE 21 +#define PragTyp_LOCKING_MODE 22 +#define PragTyp_PAGE_COUNT 23 +#define PragTyp_MMAP_SIZE 24 +#define PragTyp_OPTIMIZE 25 #define PragTyp_PAGE_SIZE 26 #define PragTyp_SECURE_DELETE 27 #define PragTyp_SHRINK_MEMORY 28 @@ -133,11 +133,6 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif - {/* zName: */ "analyze_as_needed", - /* ePragTyp: */ PragTyp_ANALYZE_AS_NEEDED, - /* ePragFlg: */ PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -419,6 +414,13 @@ static const PragmaName aPragmaName[] = { /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "page_count", /* ePragTyp: */ PragTyp_PAGE_COUNT, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 2201ea1306..f0ba49b545 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -362,7 +362,7 @@ set pragma_def { NAME: threads FLAG: Result0 - NAME: analyze_as_needed + NAME: optimize FLAG: NoColumns } From bce0414844b9ed5a99c6c2462a92d6db095b9bf2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Feb 2017 00:58:36 +0000 Subject: [PATCH 10/12] Do a single OP_Expire at the very end of "PRAGMA optimize", and omit the OP_Expire on ANALYZE commands invoked by the pragma. FossilOrigin-Name: 188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 30 +++++++++--------------------- src/pragma.c | 1 + src/sqliteInt.h | 1 + src/vdbe.c | 2 ++ 6 files changed, 23 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 508790c7bd..9a9acb5caa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sanalyze_as_needed\spragma\sto\s"optimize".\s\sEnhance\sthe\ncomment\s(which\swill\sbecome\sdocumentation,\sassuming\sthese\schanges\sland\son\strunk)\nto\sexplain\sthat\sthe\soptimize\spragma\sis\slikely\sto\sbe\senhanced\sin\svarious\sways\nin\sfuture\sreleases\sand\sthat\sapplications\sshould\snot\sdepend\supon\sthe\scurrent\nbehavior. -D 2017-02-22T20:24:10.705 +C Do\sa\ssingle\sOP_Expire\sat\sthe\svery\send\sof\s"PRAGMA\soptimize",\sand\somit\sthe\nOP_Expire\son\sANALYZE\scommands\sinvoked\sby\sthe\spragma. +D 2017-02-23T00:58:36.868 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -332,7 +332,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 -F src/analyze.c b4857ec5b46b66049e5a3cec53c071c4902a8e8b +F src/analyze.c 6d8234916c29be943e6ea28b5bef67dff98d9905 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b @@ -388,7 +388,7 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 46202f2f5ee6957ff5cba581ee3c36507685def0 +F src/pragma.c 4b32b014bb4b460bbf0103e4631809428c1ce16b F src/pragma.h d97dd835c7f4dfb6857487707313385d44fa76c0 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c @@ -400,7 +400,7 @@ F src/shell.c bf976d5301be9d8a4c52852c97909cc9a41ee20d F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h bdc181e371ea618c85f30b4c5ee4d80f4ada6ad7 +F src/sqliteInt.h df268ce1d04df042cf43b557d2309eb0b71e86c4 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -462,7 +462,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 37e95d52675bd839cc6c209f6b8d907582a27d44 +F src/vdbe.c f520378e510fd36bbf289921798dbc8f2b3dc30d F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1 @@ -1558,7 +1558,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe073905081b421405ca425ca03c5b8b0ff5f2c8 -R b1ce2ff126387ffc06c8537ddbfa9731 +P 9fced545a6f80c55d6dc4a6106cb2d3569566b3e +R 5ab35d2cee04a5e705af62d3884bb598 U drh -Z 5165d34a54df32aa272e95c6a5e5953c +Z d17aa4a4e286313b851b7686a8b74646 diff --git a/manifest.uuid b/manifest.uuid index 8e0845a1c4..7b7c25203c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fced545a6f80c55d6dc4a6106cb2d3569566b3e \ No newline at end of file +188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index a9893826e2..d1df000943 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1388,27 +1388,14 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( i==1 ) continue; /* Do not analyze the TEMP database */ analyzeDatabase(pParse, i); } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ - analyzeTable(pParse, pIdx->pTable, pIdx); - }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ - analyzeTable(pParse, pTab, 0); - } - sqlite3DbFree(db, z); - } - } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); }else{ - /* Form 3: Analyze the fully qualified table name */ + /* Form 3: Analyze the table or index named as an argument */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zDbSName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -1418,10 +1405,11 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } sqlite3DbFree(db, z); } - } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* diff --git a/src/pragma.c b/src/pragma.c index 714fb413b6..916ed4aa57 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1940,6 +1940,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); } } + sqlite3VdbeAddOp0(v, OP_Expire); break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fddfe2a57f..e95d63ec35 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1333,6 +1333,7 @@ struct sqlite3 { u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 skipBtreeMutex; /* True if no shared-cache backends */ + u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ diff --git a/src/vdbe.c b/src/vdbe.c index 59e401937f..5901ba0eb5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5523,7 +5523,9 @@ case OP_CreateTable: { /* out2 */ ** Run the SQL statement or statements specified in the P4 string. */ case OP_SqlExec: { + db->nSqlExec++; rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + db->nSqlExec--; if( rc ) goto abort_due_to_error; break; } From 555227bec07e99dddb8b9113d329e48caeba5131 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Feb 2017 02:15:33 +0000 Subject: [PATCH 11/12] Add two NEVER() operators in the sqlite3BtreeRowCountEst() routine. FossilOrigin-Name: 7a959f6d1ea038988cdb4c02d6f37abaec2580a0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 9 +++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9a9acb5caa..39c4d8e77a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\sa\ssingle\sOP_Expire\sat\sthe\svery\send\sof\s"PRAGMA\soptimize",\sand\somit\sthe\nOP_Expire\son\sANALYZE\scommands\sinvoked\sby\sthe\spragma. -D 2017-02-23T00:58:36.868 +C Add\stwo\sNEVER()\soperators\sin\sthe\ssqlite3BtreeRowCountEst()\sroutine. +D 2017-02-23T02:15:33.201 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -338,7 +338,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c a4ab1fb5cdeea88c4f76216e41cfecfa505c8c43 +F src/btree.c 19746a7db19308053ff6cb2ac002834822809e54 F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 43f903c9082040ced2b421543cb0300c2973647d @@ -1558,7 +1558,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9fced545a6f80c55d6dc4a6106cb2d3569566b3e -R 5ab35d2cee04a5e705af62d3884bb598 +P 188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 +R ea4c0d1070247cd8f7e1aa7ed210345c U drh -Z d17aa4a4e286313b851b7686a8b74646 +Z 01ff8d763fbc7e4c3e39020edd5b116b diff --git a/manifest.uuid b/manifest.uuid index 7b7c25203c..1e1b04c82c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -188300a337c87b7ee0dd1f4b9a4f1bd80e70cca4 \ No newline at end of file +7a959f6d1ea038988cdb4c02d6f37abaec2580a0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4d31661f10..788a2d4d03 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5330,8 +5330,13 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - if( pCur->eState!=CURSOR_VALID ) return -1; - if( pCur->apPage[pCur->iPage]->leaf==0 ) return -1; + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1; + for(n=1, i=0; i<=pCur->iPage; i++){ n *= pCur->apPage[i]->nCell; } From 1cfaf8eafb11a7f2a0ab8dbc47c99dd333667a19 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Mar 2017 14:17:21 +0000 Subject: [PATCH 12/12] Add an optional bitmask of allowed optimizations on the "PRAGMA optimize" command. The 0x01 bit is Debug Mode. FossilOrigin-Name: a35388eef4096c1856b025dbd90143409d4a72d3 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/pragma.c | 60 +++++++++++++++++++++++++++++++++----------- src/pragma.h | 2 +- tool/mkpragmatab.tcl | 2 +- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index bbbb0936a4..fb60f9c239 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\senhancements. -D 2017-03-02T13:22:04.261 +C Add\san\soptional\sbitmask\sof\sallowed\soptimizations\son\sthe\s"PRAGMA\soptimize"\ncommand.\s\sThe\s0x01\sbit\sis\sDebug\sMode. +D 2017-03-02T14:17:21.291 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -389,8 +389,8 @@ F src/parse.y af8830094f4aecb91cb69721f3601ad10c36abc4 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 -F src/pragma.c 4b32b014bb4b460bbf0103e4631809428c1ce16b -F src/pragma.h d97dd835c7f4dfb6857487707313385d44fa76c0 +F src/pragma.c 75fdb05838c303cf0ce5846ca011b8103531c42c +F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -1498,7 +1498,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 9b499f7301fadeddeae52b95f962ef7e5a718f50 +F tool/mkpragmatab.tcl 2ffe6d5fdc2d3381621d6c77978ba054466e757f F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1560,7 +1560,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a959f6d1ea038988cdb4c02d6f37abaec2580a0 29f54b899e5cf22ece98ab41c39c41d75a4b228d -R 6bb37f552ece3665cee38d0fc37ff06c +P c60cdb47612c05c252613e50a8ac10635469fdfe +R dbee345c7724a1e9049a9032bbb0a60b U drh -Z 8ce562b5567814952b66c349d10ab9fa +Z f9893159b19aa2a55c1ddc2fbced30b3 diff --git a/manifest.uuid b/manifest.uuid index e6cd24f62f..3384381146 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c60cdb47612c05c252613e50a8ac10635469fdfe \ No newline at end of file +a35388eef4096c1856b025dbd90143409d4a72d3 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 916ed4aa57..c5f2a77eef 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1864,34 +1864,51 @@ void sqlite3Pragma( /* ** PRAGMA optimize + ** PRAGMA optimize(MASK) ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) ** ** Attempt to optimize the database. All schemas are optimized in the first - ** form, and only the specified schema is optimized in the second form. + ** two forms, and only the specified schema is optimized in the latter two. ** ** The details of optimizations performed by this pragma does are expected ** to change and improve over time. Applications should anticipate that ** this pragma will perform new optimizations in future releases. ** - ** Argments to this pragma are currently ignored, but future enhancements - ** might make use of arguments to control which optimizations are allowed - ** or to suggest limits on how much CPU time and I/O should be expended - ** in the optimization effort. + ** The optional argument is a bitmask of optimizations to perform: ** - ** The current implementation runs ANALYZE on any tables which might have - ** benefitted from having recent statistics at some point since the start - ** of the current connection. Only tables in "schema" are analyzed in the - ** second form. In the first form, all tables except TEMP tables are - ** checked. + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** In the current implementation, a table is analyzed only if both of + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is 0x000e, which means perform all of the optimizations + ** listed above except do not set Debug Mode. New optimizations may be + ** added in future releases but they will be turned off by default. The + ** default MASK will always be 0x0e. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) The query planner used sqlite_stat1-style statistics for one or + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or ** more indexes of the table at some point during the lifetime of ** the current connection. ** - ** (2) One or more indexes of the table are currently unanalyzed OR + ** (3) One or more indexes of the table are currently unanalyzed OR ** the number of rows in the table has increased by 25 times or more ** since the last time ANALYZE was run. ** @@ -1907,7 +1924,14 @@ void sqlite3Pragma( Index *pIdx; /* An index of the table */ LogEst szThreshold; /* Size threshold above which reanalysis is needd */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xe; + } iTabCur = pParse->nTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -1932,12 +1956,18 @@ void sqlite3Pragma( if( szThreshold ){ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2, szThreshold); + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", db->aDb[iDb].zDbSName, pTab->zName); - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } } } sqlite3VdbeAddOp0(v, OP_Expire); diff --git a/src/pragma.h b/src/pragma.h index 363975df55..9b1c723b3e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -417,7 +417,7 @@ static const PragmaName aPragmaName[] = { #endif {/* zName: */ "optimize", /* ePragTyp: */ PragTyp_OPTIMIZE, - /* ePragFlg: */ PragFlg_NoColumns, + /* ePragFlg: */ PragFlg_Result1, /* ColNames: */ 0, 0, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index f0ba49b545..59b245cc76 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -363,7 +363,7 @@ set pragma_def { FLAG: Result0 NAME: optimize - FLAG: NoColumns + FLAG: Result1 } # Open the output file