From ad84bd849e00917b9007edea4521da48a6a00ed1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Nov 2019 14:01:51 +0000 Subject: [PATCH] Begin an enhancement effort for the built-in DBSTAT virtual table. FossilOrigin-Name: 9b5722f0fe666b99677e5f333dd8413aefb9ace7a461d74f6558f0ac53768719 --- manifest | 19 ++--- manifest.uuid | 2 +- src/dbstat.c | 187 ++++++++++++++++++++++++++++++++++--------------- test/stat.test | 4 +- 4 files changed, 145 insertions(+), 67 deletions(-) diff --git a/manifest b/manifest index bf60fff9f8..3b7bf3e383 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sresult\sof\ssqlite3_normalized_sql()\ssurvive\sits\sstatement\sbeing\sreprepared. -D 2019-11-19T00:13:42.428 +C Begin\san\senhancement\seffort\sfor\sthe\sbuilt-in\sDBSTAT\svirtual\stable. +D 2019-11-19T14:01:51.987 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -476,7 +476,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 1b0724e66f95f33b160b1af85caaf9cceb325d22abf39bd24df4f54a73982251 F src/date.c e1d8ac7102f3f283e63e13867acb0efa33861cf34f0faf4cdbaf9fa7a1eb7041 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 -F src/dbstat.c c12833de69cb655751487d2c5a59607e36be1c58ba1f4bd536609909ad47b319 +F src/dbstat.c 191351e68acfb71ad7adf0da464d462a4ead24284dfa81fd4f5911c6cb2e44d1 F src/delete.c a5c59b9c0251cf7682bc52af0d64f09b1aefc6781a63592c8f1136f7b73c66e4 F src/expr.c a138de8ae79628a73da2597617dbeafb0f083172be81d93ffa1cafa45161ee8c F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 @@ -1373,7 +1373,7 @@ F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test 264f4b0f941800ba139d25e33ee919c5d95fea06dfbe8ac291d6811a30984ca5 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a -F test/stat.test f8f1279ffffabe6df825723af18cc6e0ae70a893 +F test/stat.test a08fd3f87e809680824cf1940c362ee34ff83d0cda03121376dfacfebbb352e0 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5 @@ -1850,7 +1850,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 175c15008e9f19b8f6762c2fe4a545735128765081980eed01d5e46ca4acb500 -R 5a42019895e7f78a0e0e0e8c537226aa -U mistachkin -Z 324f0ce91b2d6545c1ae3061db5c270b +P 4330f0795dbc2ab41dddd41d5979331fb9b78c477c66367c4be52f929531a45f +R 3244f69d482823d13732c8928074f447 +T *branch * dbstat-enhancements +T *sym-dbstat-enhancements * +T -sym-trunk * +U drh +Z 6e47b17aa53d8cb9bfcbdbcbe411679c diff --git a/manifest.uuid b/manifest.uuid index af65542e3a..4dd1ca8028 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4330f0795dbc2ab41dddd41d5979331fb9b78c477c66367c4be52f929531a45f \ No newline at end of file +9b5722f0fe666b99677e5f333dd8413aefb9ace7a461d74f6558f0ac53768719 \ No newline at end of file diff --git a/src/dbstat.c b/src/dbstat.c index 9b72fb147c..b3523e01af 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -12,7 +12,7 @@ ** ** This file contains an implementation of the "dbstat" virtual table. ** -** The dbstat virtual table is used to extract low-level formatting +** The dbstat virtual table is used to extract low-level storage ** information from an SQLite database in order to implement the ** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script ** for an example implementation. @@ -56,27 +56,29 @@ ** ** '/1c2/000/' // Left-most child of 451st child of root */ -#define VTAB_SCHEMA \ - "CREATE TABLE xx( " \ - " name TEXT, /* Name of table or index */" \ - " path TEXT, /* Path to page from root */" \ - " pageno INTEGER, /* Page number */" \ - " pagetype TEXT, /* 'internal', 'leaf' or 'overflow' */" \ - " ncell INTEGER, /* Cells on page (0 for overflow) */" \ - " payload INTEGER, /* Bytes of payload on this page */" \ - " unused INTEGER, /* Bytes of unused space on this page */" \ - " mx_payload INTEGER, /* Largest payload size of all cells */" \ - " pgoffset INTEGER, /* Offset of page in file */" \ - " pgsize INTEGER, /* Size of the page */" \ - " schema TEXT HIDDEN /* Database schema being analyzed */" \ +#define VTAB_SCHEMA \ + "CREATE TABLE xx( " \ + " name TEXT," /* 0 Name of table or index */ \ + " path TEXT," /* 1 Path to page from root */ \ + " pageno INTEGER," /* 2 Page number */ \ + " pagetype TEXT," /* 3 'internal', 'leaf' or 'overflow' */ \ + " ncell INTEGER," /* 4 Cells on page (0 for overflow) */ \ + " payload INTEGER," /* 5 Bytes of payload on this page */ \ + " unused INTEGER," /* 6 Bytes of unused space on this page */ \ + " mx_payload INTEGER," /* 7 Largest payload size of all cells */ \ + " pgoffset INTEGER," /* 8 Offset of page in file */ \ + " pgsize INTEGER," /* 9 Size of the page */ \ + " schema TEXT HIDDEN," /* 10 Database schema being analyzed */ \ + " aggregate BOOLEAN HIDDEN" /* 11 aggregate info for each table */ \ ");" - +/* Forward reference to data structured used in this module */ typedef struct StatTable StatTable; typedef struct StatCursor StatCursor; typedef struct StatPage StatPage; typedef struct StatCell StatCell; +/* Size information for a single cell within a btree page */ struct StatCell { int nLocal; /* Bytes of local payload */ u32 iChildPg; /* Child node (or 0 if this is a leaf) */ @@ -86,10 +88,11 @@ struct StatCell { int iOvfl; /* Iterates through aOvfl[] */ }; +/* Size information for a single btree page */ struct StatPage { - u32 iPgno; - DbPage *pPg; - int iCell; + u32 iPgno; /* Page number */ + DbPage *pPg; /* Page content */ + int iCell; /* Current cell */ char *zPath; /* Path to this page */ @@ -99,16 +102,18 @@ struct StatPage { int nUnused; /* Number of unused bytes on page */ StatCell *aCell; /* Array of parsed cells */ u32 iRightChildPg; /* Right-child page number (or 0) */ - int nMxPayload; /* Largest payload of any cell on this page */ + int nMxPayload; /* Largest payload of any cell on the page */ }; +/* The cursor for scanning the dbstat virtual table */ struct StatCursor { - sqlite3_vtab_cursor base; + sqlite3_vtab_cursor base; /* base class. MUST BE FIRST! */ sqlite3_stmt *pStmt; /* Iterates through set of root pages */ int isEof; /* After pStmt has returned SQLITE_DONE */ int iDb; /* Schema used for this query */ + int isAgg; /* Aggregate results for each table */ - StatPage aPage[32]; + StatPage aPage[32]; /* Pages in path to current page */ int iPage; /* Current entry in aPage[] */ /* Values to return. */ @@ -124,9 +129,10 @@ struct StatCursor { int szPage; /* Value of 'pgSize' column */ }; +/* An instance of the DBSTAT virtual table */ struct StatTable { - sqlite3_vtab base; - sqlite3 *db; + sqlite3_vtab base; /* base class. MUST BE FIRST! */ + sqlite3 *db; /* Database connection that owns this vtab */ int iDb; /* Index of database to analyze */ }; @@ -135,7 +141,7 @@ struct StatTable { #endif /* -** Connect to or create a statvfs virtual table. +** Connect to or create a new DBSTAT virtual table. */ static int statConnect( sqlite3 *db, @@ -177,7 +183,7 @@ static int statConnect( } /* -** Disconnect from or destroy a statvfs virtual table. +** Disconnect from or destroy the DBSTAT virtual table. */ static int statDisconnect(sqlite3_vtab *pVtab){ sqlite3_free(pVtab); @@ -185,14 +191,20 @@ static int statDisconnect(sqlite3_vtab *pVtab){ } /* -** There is no "best-index". This virtual table always does a linear -** scan. However, a schema=? constraint should cause this table to -** operate on a different database schema, so check for it. +** Compute the best query strategy and return the result in idxNum. ** -** idxNum is normally 0, but will be 1 if a schema=? constraint exists. +** idxNum-Bit Meaning +** ---------- ---------------------------------------------- +** 0x01 There is a schema=? term in the WHERE clause +** 0x02 There is a name=? term in the WHERE clause +** 0x04 There is an aggregate=? term in the WHERE clause +** 0x08 Output should be ordered by name and path */ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; + int iSchema = -1; + int iName = -1; + int iAgg = -1; /* Look for a valid schema=? constraint. If found, change the idxNum to ** 1 and request the value of that constraint be sent to xFilter. And @@ -200,16 +212,43 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** used. */ for(i=0; inConstraint; i++){ - if( pIdxInfo->aConstraint[i].iColumn!=10 ) continue; - if( pIdxInfo->aConstraint[i].usable==0 ) return SQLITE_CONSTRAINT; if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - pIdxInfo->idxNum = 1; - pIdxInfo->estimatedCost = 1.0; - pIdxInfo->aConstraintUsage[i].argvIndex = 1; - pIdxInfo->aConstraintUsage[i].omit = 1; - break; + if( pIdxInfo->aConstraint[i].usable==0 ){ + /* Force DBSTAT table should always be the right-most table in a join */ + return SQLITE_CONSTRAINT; + } + switch( pIdxInfo->aConstraint[i].iColumn ){ + case 0: { /* name */ + iName = i; + break; + } + case 10: { /* schema */ + iSchema = i; + break; + } + case 11: { /* aggregate */ + iAgg = i; + break; + } + } } - + i = 0; + if( iSchema>=0 ){ + pIdxInfo->aConstraintUsage[iSchema].argvIndex = ++i; + pIdxInfo->aConstraintUsage[iSchema].omit = 1; + pIdxInfo->idxNum |= 0x01; + } + if( iName>=0 ){ + pIdxInfo->aConstraintUsage[iName].argvIndex = ++i; + pIdxInfo->aConstraintUsage[iName].omit = 1; + pIdxInfo->idxNum |= 0x02; + } + if( iAgg>=0 ){ + pIdxInfo->aConstraintUsage[iAgg].argvIndex = ++i; + pIdxInfo->aConstraintUsage[iAgg].omit = 1; + pIdxInfo->idxNum |= 0x04; + } + pIdxInfo->estimatedCost = 1.0; /* Records are always returned in ascending order of (name, path). ** If this will satisfy the client, set the orderByConsumed flag so that @@ -227,6 +266,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ) ){ pIdxInfo->orderByConsumed = 1; + pIdxInfo->idxNum |= 0x08; } return SQLITE_OK; @@ -294,11 +334,15 @@ static int statClose(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } -static void getLocalPayload( +/* +** For a single cell on a btree page, compute the number of bytes of +** content (payload) stored on that page. That is to say, compute the +** number of bytes of content not found on overflow pages. +*/ +static int getLocalPayload( int nUsable, /* Usable bytes per page */ u8 flags, /* Page flags */ - int nTotal, /* Total record (payload) size */ - int *pnLocal /* OUT: Bytes stored locally */ + int nTotal /* Total record (payload) size */ ){ int nLocal; int nMinLocal; @@ -314,7 +358,7 @@ static void getLocalPayload( nLocal = nMinLocal + (nTotal - nMinLocal) % (nUsable - 4); if( nLocal>nMaxLocal ) nLocal = nMinLocal; - *pnLocal = nLocal; + return nLocal; } static int statDecodePage(Btree *pBt, StatPage *p){ @@ -387,7 +431,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){ iOff += sqlite3GetVarint(&aData[iOff], &dummy); } if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; - getLocalPayload(nUsable, p->flags, nPayload, &nLocal); + nLocal = getLocalPayload(nUsable, p->flags, nPayload); if( nLocal<0 ) goto statPageIsCorrupt; pCell->nLocal = nLocal; assert( nPayload>=(u32)nLocal ); @@ -597,6 +641,10 @@ static int statEof(sqlite3_vtab_cursor *pCursor){ return pCsr->isEof; } +/* Initialize a cursor according to the query plan idxNum using the +** arguments in argv[0]. See statBestIndex() for a description of the +** meaning of the bits in idxNum. +*/ static int statFilter( sqlite3_vtab_cursor *pCursor, int idxNum, const char *idxStr, @@ -604,11 +652,18 @@ static int statFilter( ){ StatCursor *pCsr = (StatCursor *)pCursor; StatTable *pTab = (StatTable*)(pCursor->pVtab); - char *zSql; - int rc = SQLITE_OK; + sqlite3_str *pSql; /* Query of btrees to analyze */ + char *zSql; /* String value of pSql */ + int iArg = 0; /* Count of argv[] parameters used so far */ + int rc = SQLITE_OK; /* Result of this operation */ + const char *zName = 0; /* Only provide analysis of this table */ - if( idxNum==1 ){ - const char *zDbase = (const char*)sqlite3_value_text(argv[0]); + statResetCsr(pCsr); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( idxNum & 0x01 ){ + /* schema=? constraint is present. Get its value */ + const char *zDbase = (const char*)sqlite3_value_text(argv[iArg++]); pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase); if( pCsr->iDb<0 ){ sqlite3_free(pCursor->pVtab->zErrMsg); @@ -618,15 +673,31 @@ static int statFilter( }else{ pCsr->iDb = pTab->iDb; } - statResetCsr(pCsr); - sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - zSql = sqlite3_mprintf( - "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" - " UNION ALL " - "SELECT name, rootpage, type" - " FROM \"%w\".sqlite_master WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pCsr->iDb].zDbSName); + if( idxNum & 0x02 ){ + /* name=? constraint is present */ + zName = (const char*)sqlite3_value_text(argv[iArg++]); + } + if( idxNum & 0x04 ){ + /* aggregate=? constraint is present */ + pCsr->isAgg = sqlite3_value_double(argv[iArg++])!=0.0; + }else{ + pCsr->isAgg = 0; + } + pSql = sqlite3_str_new(pTab->db); + sqlite3_str_appendf(pSql, + "SELECT * FROM (" + "SELECT 'sqlite_master' AS name,1 AS rootpage,'table' AS type" + " UNION ALL " + "SELECT name,rootpage,type" + " FROM \"%w\".sqlite_master WHERE rootpage!=0)", + pTab->db->aDb[pCsr->iDb].zDbSName); + if( zName ){ + sqlite3_str_appendf(pSql, "WHERE name=%Q", zName); + } + if( idxNum & 0x08 ){ + sqlite3_str_appendf(pSql, " ORDER BY name"); + } + zSql = sqlite3_str_finish(pSql); if( zSql==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -677,12 +748,16 @@ static int statColumn( case 9: /* pgsize */ sqlite3_result_int(ctx, pCsr->szPage); break; - default: { /* schema */ + case 10: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); int iDb = pCsr->iDb; sqlite3_result_text(ctx, db->aDb[iDb].zDbSName, -1, SQLITE_STATIC); break; } + default: { /* aggregate */ + sqlite3_result_int(ctx, pCsr->isAgg); + break; + } } return SQLITE_OK; } diff --git a/test/stat.test b/test/stat.test index 66ca5e2f2b..f13f3ad193 100644 --- a/test/stat.test +++ b/test/stat.test @@ -108,7 +108,7 @@ do_execsql_test stat-2.1 { INSERT INTO t3 SELECT a_string(110+rowid), a_string(221+rowid) FROM t3 ORDER BY rowid; SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload - FROM stat WHERE name != 'sqlite_master'; + FROM stat WHERE name != 'sqlite_master' ORDER BY name; } [list \ sqlite_autoindex_t3_1 / 3 internal 3 368 623 125 \ sqlite_autoindex_t3_1 /000/ 8 leaf 8 946 46 123 \ @@ -150,7 +150,7 @@ do_execsql_test stat-3.1 { CREATE INDEX i4 ON t4(x); INSERT INTO t4(rowid, x) VALUES(2, a_string(7777)); SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload - FROM stat WHERE name != 'sqlite_master'; + FROM stat WHERE name != 'sqlite_master' ORDER BY name; } [list \ i4 / 3 leaf 1 103 905 7782 \ i4 /000+000000 4 overflow 0 1020 0 0 \