From da4c409aea3ccd9cc30219a67ddb3677d0d4521f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Jun 2018 18:22:10 +0000 Subject: [PATCH 1/8] Consider doing a partial table scan to fulfill an IN operator rather than using an index. Try to pick the plan with the lowest cost. FossilOrigin-Name: 1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b --- manifest | 16 +++++++++------- manifest.uuid | 2 +- src/where.c | 32 +++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f31ac5a47b..d068879da9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sIN-early-out\soptimization:\s\sWhen\sdoing\sa\slook-up\son\sa\smulti-column\sindex\nand\san\sIN\soperator\sis\sused\son\sa\scolumn\sother\sthan\sthe\sleft-most\scolumn,\sthen\nif\sno\srows\smatch\sagainst\sthe\sfirst\sIN\svalue,\scheck\sto\smake\ssure\sthere\sexist\nrows\sthat\smatch\sthe\scolumns\sto\sthe\sright\sbefore\scontinuing\swith\sthe\snext\sIN\nvalue. -D 2018-06-07T18:13:49.091 +C Consider\sdoing\sa\spartial\stable\sscan\sto\sfulfill\san\sIN\soperator\srather\nthan\susing\san\sindex.\s\sTry\sto\spick\sthe\splan\swith\sthe\slowest\scost. +D 2018-06-08T18:22:10.089 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 7dcb13bbcfd8c926546946556014c8f5aa0829eb8b65a6c18f8d187d265200a5 +F src/where.c 8e95858b398f7451c30ed05dd372f39767f04f423fa1d15a19d568268ae9a557 F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a @@ -1731,8 +1731,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb87c054b1b76959e46258ac66b24027f468b390a4148ac67f208a1fbeda4060 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55 -R fe7d026f8be3134e2723fc39eeec5ed5 -T +closed 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55 +P 09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21 +R e63ef1b65ea2f09e13127dc0f6a98d71 +T *branch * in-scan-vs-index +T *sym-in-scan-vs-index * +T -sym-trunk * U drh -Z 8a623af2a2288d1935dd9b3507daa5d8 +Z f877a3ca3a67b98847cd7a3f31dd6a75 diff --git a/manifest.uuid b/manifest.uuid index 9bc93f1fe0..6f59a009e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21 \ No newline at end of file +1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 10cb138143..9acfbcf568 100644 --- a/src/where.c +++ b/src/where.c @@ -2451,7 +2451,7 @@ static int whereLoopAddBtreeIndex( if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; + LogEst M, logK; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; @@ -2471,6 +2471,36 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ } + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the right-hand side of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + */ + M = pProbe->aiRowLogEst[saved_nEq+1]; + logK = sqlite3LogEst(nIn); + if( M + logK + 10 < nIn + rLogSize ){ + WHERETRACE(0x40, + ("IN operator costs more than scan on column %d of \"%s\" (%d<%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + continue; + }else{ + WHERETRACE(0x40, + ("IN operator cheaper than scan on column %d of \"%s\" (%d>=%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + } + pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; From f3cd0c82dfc744e446203c89c7152380f85da5d5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Jun 2018 19:13:57 +0000 Subject: [PATCH 2/8] Fix an assert() that can be false for a corrupt database and a strange query that uses a recursive SQL function to delete content from a corrupt database file while it is being queried. FossilOrigin-Name: 99057383acc8f92093530e216c621d40386a06fe98131ff0af6df524d80a6410 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/btree.c | 11 ++++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f31ac5a47b..c8299d9c0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sIN-early-out\soptimization:\s\sWhen\sdoing\sa\slook-up\son\sa\smulti-column\sindex\nand\san\sIN\soperator\sis\sused\son\sa\scolumn\sother\sthan\sthe\sleft-most\scolumn,\sthen\nif\sno\srows\smatch\sagainst\sthe\sfirst\sIN\svalue,\scheck\sto\smake\ssure\sthere\sexist\nrows\sthat\smatch\sthe\scolumns\sto\sthe\sright\sbefore\scontinuing\swith\sthe\snext\sIN\nvalue. -D 2018-06-07T18:13:49.091 +C Fix\san\sassert()\sthat\scan\sbe\sfalse\sfor\sa\scorrupt\sdatabase\sand\sa\sstrange\squery\nthat\suses\sa\srecursive\sSQL\sfunction\sto\sdelete\scontent\sfrom\sa\scorrupt\sdatabase\nfile\swhile\sit\sis\sbeing\squeried. +D 2018-06-08T19:13:57.914 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -436,7 +436,7 @@ F src/auth.c a38f3c63c974787ecf75e3213f8cac6568b9a7af7591fb0372ec0517dd16dca8 F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c ae7687faa330562d36cede3375fdc316bff068e9216322d0dfbb3749333e5d3d +F src/btree.c 06690f5ad144c5e69dfb68d5ec8ee4819c6d40f4b8bc77aa97975938f59d928c F src/btree.h ab639c4b9b210b8f4cd7a3a922af73df9a3f27c1d124267339fd73ef8619f488 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 F src/build.c 5fc41458505331bfb0c175f40b9a13cb335f826bed3ae311aaae000c132d7b16 @@ -1731,8 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb87c054b1b76959e46258ac66b24027f468b390a4148ac67f208a1fbeda4060 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55 -R fe7d026f8be3134e2723fc39eeec5ed5 -T +closed 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55 +P 09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21 +R e90ea8c0bc9f17fa862de55346ad01f3 U drh -Z 8a623af2a2288d1935dd9b3507daa5d8 +Z dc7dec1a328a3fd8e94df605e9d025f7 diff --git a/manifest.uuid b/manifest.uuid index 9bc93f1fe0..715434fba3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21 \ No newline at end of file +99057383acc8f92093530e216c621d40386a06fe98131ff0af6df524d80a6410 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 80b05d36cd..a17fbbf539 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5589,7 +5589,16 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - assert( pPage->isInit ); + if( !pPage->isInit ){ + /* The only known way for this to happen is for there to be a + ** recursive SQL function that does a DELETE operation as part of a + ** SELECT which deletes content out from under an active cursor + ** in a corrupt database file where the table being DELETE-ed from + ** has pages in common with the table being queried. See TH3 + ** module cov1/btree78.test testcase 220 (2018-06-08) for an + ** example. */ + return SQLITE_CORRUPT_BKPT; + } /* If the database file is corrupt, it is possible for the value of idx ** to be invalid here. This can only occur if a second cursor modifies From 6d6decb8d92ea6422f301ffa6dfbb1015aec8110 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Jun 2018 21:21:01 +0000 Subject: [PATCH 3/8] Only choose to scan an IN operator rather than use an index if we have real STAT1 data to suggest it is advantageous. FossilOrigin-Name: 30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/where.c | 60 ++++++++++++++++++++++++--------------------- test/in6.test | 3 +++ test/rowvalue4.test | 2 +- 5 files changed, 45 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 363f32cb2e..bb077b2153 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sbtreeNext()\sassertion\sbug\sfix\sfrom\strunk. -D 2018-06-08T19:54:07.050 +C Only\schoose\sto\sscan\san\sIN\soperator\srather\sthan\suse\san\sindex\sif\swe\shave\nreal\sSTAT1\sdata\sto\ssuggest\sit\sis\sadvantageous. +D 2018-06-08T21:21:01.417 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 8e95858b398f7451c30ed05dd372f39767f04f423fa1d15a19d568268ae9a557 +F src/where.c e4d48338ca4718c8034f313cca202cc5ca75639a24d27b959b65578198f63c81 F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a @@ -967,7 +967,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 F test/in5.test 7ae37fcd4a5e198291c6ab5f31a5bb3d15397efe8b75a6736d7a95a7b8dd9e08 -F test/in6.test 77c3e1d356d8aeb0864051f0677d3c0a032cf97b7f33a0ba8fa2b04a663f6b7b +F test/in6.test 62d943a02f722948f4410ee0b53c3cb39acd7c41afb083df8d7004238fe90a20 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1185,7 +1185,7 @@ F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d F test/rowvalue.test ef851a80f7e6de93b51caca9e4b6b7d2dcd540bbcca7d51860e80435b8b4c0de F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 -F test/rowvalue4.test cbd1cead27a797d11ec93301fd801c89e97eb1809b3d984b7f16a3876e362eac +F test/rowvalue4.test 2b20468da3775aba971caf3158e9696a4d99c69a7623fb495f332a596daebbee F test/rowvalue5.test c81c7d8cf36711ab37675ad7376084ae2a359cb6 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test 5d06ff19d9e6969e574a2e662a531dd0c67801a8 @@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b 99057383acc8f92093530e216c621d40386a06fe98131ff0af6df524d80a6410 -R 56ae5d735c58aa67ec9684ecac6c800d +P 11bd66e09035039fbfc4c5631196962dd3f2e6634f5123a7b2fec17aeaaab07b +R 54c7353f86aece043f7640340784a722 U drh -Z b50b0532bda90928b75cfc1466c6bb97 +Z b0c1273ce7490f506712c4b76e5917fd diff --git a/manifest.uuid b/manifest.uuid index 481057e482..764612de10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11bd66e09035039fbfc4c5631196962dd3f2e6634f5123a7b2fec17aeaaab07b \ No newline at end of file +30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9acfbcf568..7491027f68 100644 --- a/src/where.c +++ b/src/where.c @@ -2471,34 +2471,38 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ } - /* Let: - ** N = the total number of rows in the table - ** K = the number of entries on the right-hand side of the IN operator - ** M = the number of rows in the table that match terms to the - ** to the left in the same index. If the IN operator is on - ** the left-most index column, M==N. - ** - ** Given the definitions above, it is better to omit the IN operator - ** from the index lookup and instead do a scan of the M elements, - ** testing each scanned row against the IN operator separately, if: - ** - ** M*log(K) < K*log(N) - ** - ** Our estimates for M, K, and N might be inaccurate, so we build in - ** a safety margin of 2 (LogEst: 10) that favors using the IN operator - ** with the index, as using an index has better worst-case behavior. - */ - M = pProbe->aiRowLogEst[saved_nEq+1]; - logK = sqlite3LogEst(nIn); - if( M + logK + 10 < nIn + rLogSize ){ - WHERETRACE(0x40, - ("IN operator costs more than scan on column %d of \"%s\" (%d<%d)\n", - saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); - continue; - }else{ - WHERETRACE(0x40, - ("IN operator cheaper than scan on column %d of \"%s\" (%d>=%d)\n", - saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + if( pProbe->hasStat1 ){ + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the RHS of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + ** If we do not have real sqlite_stat1 data, always prefer to use + ** the index. + */ + M = pProbe->aiRowLogEst[saved_nEq]; + logK = estLog(nIn); + if( M + logK + 10 < nIn + rLogSize ){ + WHERETRACE(0x40, + ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + continue; + }else{ + WHERETRACE(0x40, + ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + } } pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ diff --git a/test/in6.test b/test/in6.test index ad16c4e8a8..773ee589d4 100644 --- a/test/in6.test +++ b/test/in6.test @@ -28,6 +28,9 @@ do_test in6-1.1 { INSERT INTO t1(a,b,c,d) SELECT 100, 200+x/2, 300+x/5, x FROM c; CREATE INDEX t1abc ON t1(a,b,c); + ANALYZE; + UPDATE sqlite_stat1 SET stat='1000000 500000 500 50'; + ANALYZE sqlite_master; } set ::sqlite_search_count 0 db eval { diff --git a/test/rowvalue4.test b/test/rowvalue4.test index ce025fc923..5c0d170f59 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -224,7 +224,7 @@ do_execsql_test 5.0 { WITH i(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM i WHERE i<1000 ) - INSERT INTO d2 SELECT i/3, i%3, i/3 FROM i; + INSERT INTO d2 SELECT i/100, i%100, i/100 FROM i; ANALYZE; } From da230bd484d63bd6adff862b74f9090cc921ad24 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jun 2018 00:09:58 +0000 Subject: [PATCH 4/8] Avoid invoking the whereLoopAddOr() routine in the query planner if there are no OR operators in the WHERE clause, thus speeding up query planning slightly. FossilOrigin-Name: 292724ffc4bfca435fff309383d488ffdbe1e314e5eb26da21cf2f621b64bce5 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/where.c | 2 +- src/whereInt.h | 1 + src/whereexpr.c | 7 ++++++- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 61b07880e0..4a1d9dc884 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\squery\splanner\shas\sthe\sopportunity\sto\suse\san\sIN\soperater\sconstraint\non\sa\sterm\sof\san\sindex\sother\sthan\sthe\sleft-most\sterm,\suse\sthe\sestimated\snumber\nof\selements\son\sthe\sright-hand\sside\sof\sthe\sIN\soperator\sto\sdetermine\sif\smakes\nsense\sto\suse\sthe\sIN\soperator\swith\sindex\slooks,\sor\sto\sjust\sdo\sa\sscan\sover\sthe\nrange\sof\sthe\stable\sidentified\sby\sthe\sindex\sterms\sto\sthe\sleft.\s\s\sOnly\sdo\sthis\nif\ssqlite_stat1\smeasurements\sare\savailable\sas\sotherwise\sthe\sperformance\nestimates\swill\snot\sbe\saccurate\senough\sto\sdiscern\sthe\sbest\splan.\s\sBias\sthe\ndecision\sslightly\sin\sfavor\sof\susing\sindex\slookups\son\seach\selement\sof\sthe\sIN\noperator. -D 2018-06-08T23:23:53.721 +C Avoid\sinvoking\sthe\swhereLoopAddOr()\sroutine\sin\sthe\squery\splanner\sif\sthere\nare\sno\sOR\soperators\sin\sthe\sWHERE\sclause,\sthus\sspeeding\sup\squery\splanning\nslightly. +D 2018-06-09T00:09:58.256 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -579,10 +579,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c e4d48338ca4718c8034f313cca202cc5ca75639a24d27b959b65578198f63c81 -F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6 +F src/where.c b93db8c543ce60bdeb325250665e8fb5374c02b31d581c339ae8723da9b7f889 +F src/whereInt.h 09dbf692741bb5ac4f4ce15e3e124bcff3c1077dd39afd2aa7ef573040252c11 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 -F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a +F src/whereexpr.c d979cd594c9abbf038b8671a9b05f1e9d02300b06f987f2ea2c06712c60b8077 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1731,8 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 99057383acc8f92093530e216c621d40386a06fe98131ff0af6df524d80a6410 30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d -R 54c7353f86aece043f7640340784a722 -T +closed 30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d +P 2cbbabdf5ef624d809fbb40d2d312a29e0b5f02756fc0dbf6985fc8b0c8d1ade +R 0b684b2e3c17dc858ffe6bf2e9f152c7 U drh -Z 3947665ab12ebd8b4ea54e65ad1b9691 +Z 039981a09de1ec68f373ff429bf5bde6 diff --git a/manifest.uuid b/manifest.uuid index 49cd303dad..5cb8cc6192 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cbbabdf5ef624d809fbb40d2d312a29e0b5f02756fc0dbf6985fc8b0c8d1ade \ No newline at end of file +292724ffc4bfca435fff309383d488ffdbe1e314e5eb26da21cf2f621b64bce5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7491027f68..7aba0eb5dc 100644 --- a/src/where.c +++ b/src/where.c @@ -3534,7 +3534,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ { rc = whereLoopAddBtree(pBuilder, mPrereq); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); } mPrior |= pNew->maskSelf; diff --git a/src/whereInt.h b/src/whereInt.h index 5028793bb8..c15264a9be 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -322,6 +322,7 @@ struct WhereClause { WhereInfo *pWInfo; /* WHERE clause processing context */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ + u8 hasOr; /* True if any a[].eOperator is WO_OR */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ diff --git a/src/whereexpr.c b/src/whereexpr.c index 29750080a1..a0f4ff6af5 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -672,7 +672,12 @@ static void exprAnalyzeOrTerm( ** empty. */ pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; + if( indexable ){ + pTerm->eOperator = WO_OR; + pWC->hasOr = 1; + }else{ + pTerm->eOperator = WO_OR; + } /* For a two-way OR, attempt to implementation case 2. */ From 1fe3ac73474a71583b9bd05a5e9016855748c0b3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jun 2018 01:12:08 +0000 Subject: [PATCH 5/8] Compute the bitmask of indexed columns for each index once when the Index objecct is constructed, instead of recomputing it every time it is needed. FossilOrigin-Name: d735872ec383bbd220b08c61d25db9ff3675d2542b9e7867e7d6323a12e0cc23 --- manifest | 16 +++++++-------- manifest.uuid | 2 +- src/build.c | 27 +++++++++++++++++++++++++ src/sqliteInt.h | 52 +++++++++++++++++++++++++------------------------ src/where.c | 22 ++------------------- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index 4a1d9dc884..310f7a57fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sinvoking\sthe\swhereLoopAddOr()\sroutine\sin\sthe\squery\splanner\sif\sthere\nare\sno\sOR\soperators\sin\sthe\sWHERE\sclause,\sthus\sspeeding\sup\squery\splanning\nslightly. -D 2018-06-09T00:09:58.256 +C Compute\sthe\sbitmask\sof\sindexed\scolumns\sfor\seach\sindex\sonce\swhen\sthe\sIndex\nobjecct\sis\sconstructed,\sinstead\sof\srecomputing\sit\severy\stime\sit\sis\sneeded. +D 2018-06-09T01:12:08.387 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -439,7 +439,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 06690f5ad144c5e69dfb68d5ec8ee4819c6d40f4b8bc77aa97975938f59d928c F src/btree.h ab639c4b9b210b8f4cd7a3a922af73df9a3f27c1d124267339fd73ef8619f488 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 5fc41458505331bfb0c175f40b9a13cb335f826bed3ae311aaae000c132d7b16 +F src/build.c 3b3bfa88800739e1f11313dcecfba5ef8e4757b6c929cdf7de9fcfc01002b81f F src/callback.c 36caff1e7eb7deb58572d59c41cee8f064a11d00297616995c5050ea0cfc1288 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 849d4cebe008cfc6e4799b034a172b4eaf8856b100739632a852732ba66eee48 @@ -500,7 +500,7 @@ F src/shell.c.in 4d0ddf10c403710d241bf920163dcf032c21119aebb61e70840942c0eafecdf F src/sqlite.h.in 63b07f76731f2b1e55c48fdb9f0508dcc6fbe3971010b8612ffd847c3c56d9a1 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9887b27e69c01e79c2cbe74ef73bf01af5b5703d6a7f0a4371e386d7249cb1c7 -F src/sqliteInt.h d2bd297dba08f2390a91c31ff775e0964e9663df5b2910a569fe6f830b8b2beb +F src/sqliteInt.h 9332b820382288dde7d6ea712bbcd34380bbbeb44f6f7032710bd5240fd4067d F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c b93db8c543ce60bdeb325250665e8fb5374c02b31d581c339ae8723da9b7f889 +F src/where.c 9915929594a8c6bf8ca8d6e02a4f7e632948fa523712ab647490d18855a43656 F src/whereInt.h 09dbf692741bb5ac4f4ce15e3e124bcff3c1077dd39afd2aa7ef573040252c11 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 F src/whereexpr.c d979cd594c9abbf038b8671a9b05f1e9d02300b06f987f2ea2c06712c60b8077 @@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2cbbabdf5ef624d809fbb40d2d312a29e0b5f02756fc0dbf6985fc8b0c8d1ade -R 0b684b2e3c17dc858ffe6bf2e9f152c7 +P 292724ffc4bfca435fff309383d488ffdbe1e314e5eb26da21cf2f621b64bce5 +R 84501035b6f16659219d9bb1862a6d24 U drh -Z 039981a09de1ec68f373ff429bf5bde6 +Z ca51630bcd989bed6d96c283f487ffd2 diff --git a/manifest.uuid b/manifest.uuid index 5cb8cc6192..dd0ca06fe8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -292724ffc4bfca435fff309383d488ffdbe1e314e5eb26da21cf2f621b64bce5 \ No newline at end of file +d735872ec383bbd220b08c61d25db9ff3675d2542b9e7867e7d6323a12e0cc23 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 3d982f72ea..44fc4573b2 100644 --- a/src/build.c +++ b/src/build.c @@ -1695,6 +1695,31 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ return 0; } +/* Recompute the colNotIdxed field of the Index. +** +** colNotIdxed is a bitmask that has a 0 bit representing each indexed +** columns that are within the first 63 columns of the table. The +** high-order bit of colNotIdxed is always 1. All unindexed columns +** of the table have a 1. +** +** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask +** to determine if the index is covering index. +*/ +static void recomputeColumnsNotIndexed(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xcolNotIdxed = ~m; + assert( (pIdx->colNotIdxed>>63)==1 ); +} + /* ** This routine runs at the end of parsing a CREATE TABLE statement that ** has a WITHOUT ROWID clause. The job of this routine is to convert both @@ -1843,6 +1868,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ }else{ pPk->nColumn = pTab->nCol; } + recomputeColumnsNotIndexed(pPk); } /* @@ -3276,6 +3302,7 @@ void sqlite3CreateIndex( ** it as a covering index */ assert( HasRowid(pTab) || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + recomputeColumnsNotIndexed(pIndex); if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ pIndex->isCovering = 1; for(j=0; jnCol; j++){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 91fde727a2..98bd7e4079 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1109,6 +1109,32 @@ typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; typedef struct With With; + +/* +** The bitmask datatype defined below is used for various optimizations. +** +** Changing this from a 64-bit to a 32-bit type limits the number of +** tables in a join to 32 instead of 64. But it also reduces the size +** of the library by 738 bytes on ix86. +*/ +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif + +/* +** The number of bits in a Bitmask. "BMS" means "BitMask Size". +*/ +#define BMS ((int)(sizeof(Bitmask)*8)) + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define ALLBITS ((Bitmask)-1) + /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer ** variable number associated with that parameter. See the format description @@ -2198,6 +2224,7 @@ struct Index { tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif + Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ }; /* @@ -2543,31 +2570,6 @@ struct IdList { int nId; /* Number of identifiers on the list */ }; -/* -** The bitmask datatype defined below is used for various optimizations. -** -** Changing this from a 64-bit to a 32-bit type limits the number of -** tables in a join to 32 instead of 64. But it also reduces the size -** of the library by 738 bytes on ix86. -*/ -#ifdef SQLITE_BITMASK_TYPE - typedef SQLITE_BITMASK_TYPE Bitmask; -#else - typedef u64 Bitmask; -#endif - -/* -** The number of bits in a Bitmask. "BMS" means "BitMask Size". -*/ -#define BMS ((int)(sizeof(Bitmask)*8)) - -/* -** A bit in a Bitmask -*/ -#define MASKBIT(n) (((Bitmask)1)<<(n)) -#define MASKBIT32(n) (((unsigned int)1)<<(n)) -#define ALLBITS ((Bitmask)-1) - /* ** The following structure describes the FROM clause of a SELECT statement. ** Each table or subquery in the FROM clause is a separate element of diff --git a/src/where.c b/src/where.c index 7aba0eb5dc..56d37be012 100644 --- a/src/where.c +++ b/src/where.c @@ -2734,24 +2734,6 @@ static int indexMightHelpWithOrderBy( return 0; } -/* -** Return a bitmask where 1s indicate that the corresponding column of -** the table is used by an index. Only the first 63 columns are considered. -*/ -static Bitmask columnsInIndex(Index *pIdx){ - Bitmask m = 0; - int j; - for(j=pIdx->nColumn-1; j>=0; j--){ - int x = pIdx->aiColumn[j]; - if( x>=0 ){ - testcase( x==BMS-1 ); - testcase( x==BMS-2 ); - if( xwsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; }else{ - m = pSrc->colUsed & ~columnsInIndex(pProbe); + m = pSrc->colUsed & pProbe->colNotIdxed; pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } @@ -4370,7 +4352,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ } if( j!=pIdx->nKeyCol ) continue; pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; - if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ pLoop->wsFlags |= WHERE_IDX_ONLY; } pLoop->nLTerm = j; From ccf6db5e6499f26c0463ed9840209c47c1d4a7a9 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jun 2018 02:49:11 +0000 Subject: [PATCH 6/8] Performance improvement to sqlite3WhereExprUsage(). FossilOrigin-Name: fd0934139076848f2f2edfd3d74d54608531031a05a60ca6ac1d7016dcd538df --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/whereInt.h | 1 + src/whereexpr.c | 16 ++++++++++------ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 310f7a57fb..735c33f57b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Compute\sthe\sbitmask\sof\sindexed\scolumns\sfor\seach\sindex\sonce\swhen\sthe\sIndex\nobjecct\sis\sconstructed,\sinstead\sof\srecomputing\sit\severy\stime\sit\sis\sneeded. -D 2018-06-09T01:12:08.387 +C Performance\simprovement\sto\ssqlite3WhereExprUsage(). +D 2018-06-09T02:49:11.362 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -580,9 +580,9 @@ F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 9915929594a8c6bf8ca8d6e02a4f7e632948fa523712ab647490d18855a43656 -F src/whereInt.h 09dbf692741bb5ac4f4ce15e3e124bcff3c1077dd39afd2aa7ef573040252c11 +F src/whereInt.h b90ef9b9707ef750eab2a7a080c48fb4900315033274689def32d0cf5a81ebe4 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 -F src/whereexpr.c d979cd594c9abbf038b8671a9b05f1e9d02300b06f987f2ea2c06712c60b8077 +F src/whereexpr.c 6f022d6cc9daf56495f191b199352f783aff5cf268ba136b4d8cea3fb62d8c7d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 292724ffc4bfca435fff309383d488ffdbe1e314e5eb26da21cf2f621b64bce5 -R 84501035b6f16659219d9bb1862a6d24 +P d735872ec383bbd220b08c61d25db9ff3675d2542b9e7867e7d6323a12e0cc23 +R 2ba2d88f21c46106179a39d7ea349a11 U drh -Z ca51630bcd989bed6d96c283f487ffd2 +Z 5dc11e9a5cd95e10cd947ae661eb7994 diff --git a/manifest.uuid b/manifest.uuid index dd0ca06fe8..c6ce388e64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d735872ec383bbd220b08c61d25db9ff3675d2542b9e7867e7d6323a12e0cc23 \ No newline at end of file +fd0934139076848f2f2edfd3d74d54608531031a05a60ca6ac1d7016dcd538df \ No newline at end of file diff --git a/src/whereInt.h b/src/whereInt.h index c15264a9be..8c87effac0 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -496,6 +496,7 @@ void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); void sqlite3WhereClauseClear(WhereClause*); void sqlite3WhereSplit(WhereClause*,Expr*,u8); Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +Bitmask sqlite3WhereExprUsageNN(WhereMaskSet*, Expr*); Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); diff --git a/src/whereexpr.c b/src/whereexpr.c index a0f4ff6af5..3f349134ba 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1016,7 +1016,7 @@ static void exprAnalyze( pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); } pMaskSet->bVarSelect = 0; - prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); + prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); if( pMaskSet->bVarSelect ) pTerm->wtFlags |= TERM_VARSELECT; if( ExprHasProperty(pExpr, EP_FromJoin) ){ Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); @@ -1445,17 +1445,18 @@ void sqlite3WhereClauseClear(WhereClause *pWC){ ** a bitmask indicating which tables are used in that expression ** tree. */ -Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ +Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ Bitmask mask; - if( p==0 ) return 0; if( p->op==TK_COLUMN ){ return sqlite3WhereGetMask(pMaskSet, p->iTable); + }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + assert( p->op!=TK_IF_NULL_ROW ); + return 0; } mask = (p->op==TK_IF_NULL_ROW) ? sqlite3WhereGetMask(pMaskSet, p->iTable) : 0; - assert( !ExprHasProperty(p, EP_TokenOnly) ); - if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); + if( p->pLeft ) mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pLeft); if( p->pRight ){ - mask |= sqlite3WhereExprUsage(pMaskSet, p->pRight); + mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pRight); assert( p->x.pList==0 ); }else if( ExprHasProperty(p, EP_xIsSelect) ){ if( ExprHasProperty(p, EP_VarSelect) ) pMaskSet->bVarSelect = 1; @@ -1465,6 +1466,9 @@ Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ } return mask; } +Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + return p ? sqlite3WhereExprUsageNN(pMaskSet,p) : 0; +} Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ int i; Bitmask mask = 0; From a46ba6cca5102ade4689faeb67a499d56360ebb7 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jun 2018 14:13:46 +0000 Subject: [PATCH 7/8] Improved comments an presentation for the recent IN operator decision improvement. FossilOrigin-Name: 31e480f68dfd887cfd9114f9f9fec53d751d3ecc27e36f55c0166b51b2fbb08c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 735c33f57b..2408ba2066 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sto\ssqlite3WhereExprUsage(). -D 2018-06-09T02:49:11.362 +C Improved\scomments\san\spresentation\sfor\sthe\srecent\sIN\soperator\sdecision\nimprovement. +D 2018-06-09T14:13:46.495 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 9915929594a8c6bf8ca8d6e02a4f7e632948fa523712ab647490d18855a43656 +F src/where.c fe1a6f97c12cc9472ccce86166ba3f827cf61d6ae770c036a6396b63863baac4 F src/whereInt.h b90ef9b9707ef750eab2a7a080c48fb4900315033274689def32d0cf5a81ebe4 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 F src/whereexpr.c 6f022d6cc9daf56495f191b199352f783aff5cf268ba136b4d8cea3fb62d8c7d @@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d735872ec383bbd220b08c61d25db9ff3675d2542b9e7867e7d6323a12e0cc23 -R 2ba2d88f21c46106179a39d7ea349a11 +P fd0934139076848f2f2edfd3d74d54608531031a05a60ca6ac1d7016dcd538df +R b9ed4417ad4e47d1ad0ea2ffe1b71005 U drh -Z 5dc11e9a5cd95e10cd947ae661eb7994 +Z fc8a0aeffe4e41486a64c0cdfcf48c75 diff --git a/manifest.uuid b/manifest.uuid index c6ce388e64..e6f6c75858 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd0934139076848f2f2edfd3d74d54608531031a05a60ca6ac1d7016dcd538df \ No newline at end of file +31e480f68dfd887cfd9114f9f9fec53d751d3ecc27e36f55c0166b51b2fbb08c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 56d37be012..0e47dbf7ee 100644 --- a/src/where.c +++ b/src/where.c @@ -2451,7 +2451,6 @@ static int whereLoopAddBtreeIndex( if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; - LogEst M, logK; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; @@ -2472,6 +2471,7 @@ static int whereLoopAddBtreeIndex( ** changes "x IN (?)" into "x=?". */ } if( pProbe->hasStat1 ){ + LogEst M, logK, safetyMargin; /* Let: ** N = the total number of rows in the table ** K = the number of entries on the RHS of the IN operator @@ -2493,7 +2493,8 @@ static int whereLoopAddBtreeIndex( */ M = pProbe->aiRowLogEst[saved_nEq]; logK = estLog(nIn); - if( M + logK + 10 < nIn + rLogSize ){ + safetyMargin = 10; /* TUNING: extra weight for indexed IN */ + if( M + logK + safetyMargin < nIn + rLogSize ){ WHERETRACE(0x40, ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n", saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); From 4ece2f265736090c33cb0f18f5fcbfb148a9fbde Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jun 2018 16:49:00 +0000 Subject: [PATCH 8/8] Slightly smaller and faster code by encapsulating wal-index hash table location information in a separate WalHashLoc object rather than passing around the various elements as separate variables. FossilOrigin-Name: 538a365b7a32ab7fa84f59d7556242cfb59b76d287b6417eb3a823197a354e8e --- manifest | 12 ++--- manifest.uuid | 2 +- src/wal.c | 143 ++++++++++++++++++++++++-------------------------- 3 files changed, 77 insertions(+), 80 deletions(-) diff --git a/manifest b/manifest index 2408ba2066..3e06caf3c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\san\spresentation\sfor\sthe\srecent\sIN\soperator\sdecision\nimprovement. -D 2018-06-09T14:13:46.495 +C Slightly\ssmaller\sand\sfaster\scode\sby\sencapsulating\swal-index\shash\stable\nlocation\sinformation\sin\sa\sseparate\sWalHashLoc\sobject\srather\sthan\spassing\naround\sthe\svarious\selements\sas\sseparate\svariables. +D 2018-06-09T16:49:00.305 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -576,7 +576,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 10ea07dec111de6fb0a4fc87a7ffa4c65fdc088a19dbfaf7d6f2b128f2e8eb7b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 +F src/wal.c d44a0811afd2155b1157c38b33141d4ac028fda6232485bed664015bb05819ca F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c fe1a6f97c12cc9472ccce86166ba3f827cf61d6ae770c036a6396b63863baac4 @@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fd0934139076848f2f2edfd3d74d54608531031a05a60ca6ac1d7016dcd538df -R b9ed4417ad4e47d1ad0ea2ffe1b71005 +P 31e480f68dfd887cfd9114f9f9fec53d751d3ecc27e36f55c0166b51b2fbb08c +R 6ae6039dbf6c75d53dff2e4ab7fb3afb U drh -Z fc8a0aeffe4e41486a64c0cdfcf48c75 +Z 6ec430bc7e585373e73f8d45e5d8a30b diff --git a/manifest.uuid b/manifest.uuid index e6f6c75858..15b5d5a674 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31e480f68dfd887cfd9114f9f9fec53d751d3ecc27e36f55c0166b51b2fbb08c \ No newline at end of file +538a365b7a32ab7fa84f59d7556242cfb59b76d287b6417eb3a823197a354e8e \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 69a89a709b..885a2bd733 100644 --- a/src/wal.c +++ b/src/wal.c @@ -880,48 +880,51 @@ static int walNextHash(int iPriorHash){ return (iPriorHash+1)&(HASHTABLE_NSLOT-1); } +/* +** An instance of the WalHashLoc object is used to describe the location +** of a page hash table in the wal-index. This becomes the return value +** from walHashGet(). +*/ +typedef struct WalHashLoc WalHashLoc; +struct WalHashLoc { + volatile ht_slot *aHash; /* Start of the wal-index hash table */ + volatile u32 *aPgno; /* aPgno[1] is the page of first frame indexed */ + u32 iZero; /* One less than the frame number of first indexed*/ +}; + /* ** Return pointers to the hash table and page number array stored on ** page iHash of the wal-index. The wal-index is broken into 32KB pages ** numbered starting from 0. ** -** Set output variable *paHash to point to the start of the hash table -** in the wal-index file. Set *piZero to one less than the frame +** Set output variable pLoc->aHash to point to the start of the hash table +** in the wal-index file. Set pLoc->iZero to one less than the frame ** number of the first frame indexed by this hash table. If a ** slot in the hash table is set to N, it refers to frame number -** (*piZero+N) in the log. +** (pLoc->iZero+N) in the log. ** -** Finally, set *paPgno so that *paPgno[1] is the page number of the -** first frame indexed by the hash table, frame (*piZero+1). +** Finally, set pLoc->aPgno so that pLoc->aPgno[1] is the page number of the +** first frame indexed by the hash table, frame (pLoc->iZero+1). */ static int walHashGet( Wal *pWal, /* WAL handle */ int iHash, /* Find the iHash'th table */ - volatile ht_slot **paHash, /* OUT: Pointer to hash index */ - volatile u32 **paPgno, /* OUT: Pointer to page number array */ - u32 *piZero /* OUT: Frame associated with *paPgno[0] */ + WalHashLoc *pLoc /* OUT: Hash table location */ ){ int rc; /* Return code */ - volatile u32 *aPgno; - rc = walIndexPage(pWal, iHash, &aPgno); + rc = walIndexPage(pWal, iHash, &pLoc->aPgno); assert( rc==SQLITE_OK || iHash>0 ); if( rc==SQLITE_OK ){ - u32 iZero; - volatile ht_slot *aHash; - - aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; + pLoc->aHash = (volatile ht_slot *)&pLoc->aPgno[HASHTABLE_NPAGE]; if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; - iZero = 0; + pLoc->aPgno = &pLoc->aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + pLoc->iZero = 0; }else{ - iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + pLoc->iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; } - - *paPgno = &aPgno[-1]; - *paHash = aHash; - *piZero = iZero; + pLoc->aPgno = &pLoc->aPgno[-1]; } return rc; } @@ -967,9 +970,7 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ ** actually needed. */ static void walCleanupHash(Wal *pWal){ - volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ - volatile u32 *aPgno = 0; /* Page number array for hash table */ - u32 iZero = 0; /* frame == (aHash[x]+iZero) */ + WalHashLoc sLoc; /* Hash table location */ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ @@ -987,24 +988,24 @@ static void walCleanupHash(Wal *pWal){ */ assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame. */ - iLimit = pWal->hdr.mxFrame - iZero; + iLimit = pWal->hdr.mxFrame - sLoc.iZero; assert( iLimit>0 ); for(i=0; iiLimit ){ - aHash[i] = 0; + if( sLoc.aHash[i]>iLimit ){ + sLoc.aHash[i] = 0; } } /* Zero the entries in the aPgno array that correspond to frames with ** frame numbers greater than pWal->hdr.mxFrame. */ - nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); - memset((void *)&aPgno[iLimit+1], 0, nByte); + nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit+1]); + memset((void *)&sLoc.aPgno[iLimit+1], 0, nByte); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the every entry in the mapping region is still reachable @@ -1014,10 +1015,10 @@ static void walCleanupHash(Wal *pWal){ int j; /* Loop counter */ int iKey; /* Hash key */ for(j=1; j<=iLimit; j++){ - for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==j ) break; + for(iKey=walHash(sLoc.aPgno[j]);sLoc.aHash[iKey];iKey=walNextHash(iKey)){ + if( sLoc.aHash[iKey]==j ) break; } - assert( aHash[iKey]==j ); + assert( sLoc.aHash[iKey]==j ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -1030,11 +1031,9 @@ static void walCleanupHash(Wal *pWal){ */ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int rc; /* Return code */ - u32 iZero = 0; /* One less than frame number of aPgno[1] */ - volatile u32 *aPgno = 0; /* Page number array */ - volatile ht_slot *aHash = 0; /* Hash table */ + WalHashLoc sLoc; /* Wal-index hash table location */ - rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(iFrame), &sLoc); /* Assuming the wal-index file was successfully mapped, populate the ** page number array and hash table entry. @@ -1044,15 +1043,16 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int idx; /* Value to write to hash-table slot */ int nCollide; /* Number of hash collisions */ - idx = iFrame - iZero; + idx = iFrame - sLoc.iZero; assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the ** entire hash table and aPgno[] array before proceeding. */ if( idx==1 ){ - int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); - memset((void*)&aPgno[1], 0, nByte); + int nByte = (int)((u8 *)&sLoc.aHash[HASHTABLE_NSLOT] + - (u8 *)&sLoc.aPgno[1]); + memset((void*)&sLoc.aPgno[1], 0, nByte); } /* If the entry in aPgno[] is already set, then the previous writer @@ -1061,18 +1061,18 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ ** Remove the remnants of that writers uncommitted transaction from ** the hash-table before writing any new entries. */ - if( aPgno[idx] ){ + if( sLoc.aPgno[idx] ){ walCleanupHash(pWal); - assert( !aPgno[idx] ); + assert( !sLoc.aPgno[idx] ); } /* Write the aPgno[] array entry and the hash-table slot. */ nCollide = idx; - for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - aPgno[idx] = iPage; - aHash[iKey] = (ht_slot)idx; + sLoc.aPgno[idx] = iPage; + sLoc.aHash[iKey] = (ht_slot)idx; #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the number of entries in the hash table exactly equals @@ -1081,7 +1081,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ { int i; /* Loop counter */ int nEntry = 0; /* Number of entries in the hash table */ - for(i=0; iaSegment[p->nSegment])[iZero]; - iZero++; + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero]; + sLoc.iZero++; for(j=0; jaSegment[i].iZero = iZero; + walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry); + p->aSegment[i].iZero = sLoc.iZero; p->aSegment[i].nEntry = nEntry; p->aSegment[i].aIndex = aIndex; - p->aSegment[i].aPgno = (u32 *)aPgno; + p->aSegment[i].aPgno = (u32 *)sLoc.aPgno; } } sqlite3_free(aTmp); @@ -2673,16 +2673,14 @@ int sqlite3WalSnapshotRecover(Wal *pWal){ }else{ u32 i = pInfo->nBackfillAttempted; for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ - volatile ht_slot *dummy; - volatile u32 *aPgno; /* Array of page numbers */ - u32 iZero; /* Frame corresponding to aPgno[0] */ + WalHashLoc sLoc; /* Hash table location */ u32 pgno; /* Page number in db file */ i64 iDbOff; /* Offset of db file entry */ i64 iWalOff; /* Offset of wal file entry */ - rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(i), &sLoc); if( rc!=SQLITE_OK ) break; - pgno = aPgno[i-iZero]; + pgno = sLoc.aPgno[i-sLoc.iZero]; iDbOff = (i64)(pgno-1) * szPage; if( iDbOff+szPage<=szDb ){ @@ -2883,21 +2881,20 @@ int sqlite3WalFindFrame( */ iMinHash = walFramePage(pWal->minFrame); for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ - volatile ht_slot *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ - u32 iZero; /* Frame number corresponding to aPgno[0] */ + WalHashLoc sLoc; /* Hash table location */ int iKey; /* Hash slot index */ int nCollide; /* Number of hash collisions remaining */ int rc; /* Error code */ - rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, iHash, &sLoc); if( rc!=SQLITE_OK ){ return rc; } nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ + u32 iFrame = sLoc.aHash[iKey] + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame + && sLoc.aPgno[sLoc.aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; }