From 6f654a40e8618e41b96be3d8dc36cac8ed69a56c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Apr 2017 19:59:55 +0000 Subject: [PATCH 1/2] Within a loop that uses a non-covering index test, test non-indexed terms that can be tested without seeking the main table cursor before those that cannot. FossilOrigin-Name: afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 --- manifest | 16 ++++++---- manifest.uuid | 2 +- src/wherecode.c | 78 ++++++++++++++++++++++++++-------------------- test/pushdown.test | 59 +++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 40 deletions(-) create mode 100644 test/pushdown.test diff --git a/manifest b/manifest index 8c73ad3567..062de1d7ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\scachespill.test. -D 2017-04-26T17:21:33.779 +C Within\sa\sloop\sthat\suses\sa\snon-covering\sindex\stest,\stest\snon-indexed\sterms\sthat\ncan\sbe\stested\swithout\sseeking\sthe\smain\stable\scursor\sbefore\sthose\sthat\scannot. +D 2017-04-28T19:59:55.785 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -485,7 +485,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 -F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681 +F src/wherecode.c 878a8c75af2bf6103ed4d1df639ee64ddb66f11b3a388f9caf7598c159a2c9af F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1043,6 +1043,7 @@ F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc +F test/pushdown.test a5d2e5e66cc94cfb0989a5dd23e77427dd6c4313019155ba9051f52dfc5326aa F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 @@ -1576,7 +1577,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e -R c4490df1c59aee5f8635e7285ee6a7e5 +P 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 +R 8c650dff0f0f10ff63f37c52f3854c68 +T *branch * pushdown-optimization +T *sym-pushdown-optimization * +T -sym-trunk * U dan -Z 2b0ae6051b3e0017c3cacc8dc5ffb6e4 +Z 464ba506caa7cc6d49be35980d40d709 diff --git a/manifest.uuid b/manifest.uuid index 44877c7f80..0b47274c6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 \ No newline at end of file +afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index d429fbb523..a95889136a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1129,6 +1129,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Index *pIdx = 0; /* Index used by loop (if any) */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -1454,7 +1455,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( int endEq; /* True if range end uses ==, >= or <= */ int start_constraints; /* Start of range is constrained */ int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ int iIdxCur; /* The VDBE cursor for the index */ int nExtraReg = 0; /* Number of extra registers needed */ int op; /* Instruction opcode */ @@ -1705,6 +1705,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( }else{ assert( pLevel->p5==0 ); } + if( omitTable ) pIdx = 0; }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION @@ -2022,42 +2023,53 @@ Bitmask sqlite3WhereCodeOneLoopStart( /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. - */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - int skipLikeAddr = 0; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - if( pTerm->wtFlags & TERM_LIKECOND ){ - /* If the TERM_LIKECOND flag is set, that means that the range search - ** is sufficient to guarantee that the LIKE operator is true, so we - ** can skip the call to the like(A,B) function. But this only works - ** for strings. So do not skip the call to the function on the pass - ** that compares BLOBs. */ + ** + ** This loop may run either once (pIdx==0) or twice (pIdx!=0). If + ** it is run twice, then the first iteration codes those sub-expressions + ** that can be computed using columns from pIdx only (without seeking + ** the main table cursor). */ + while( 1 ){ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + continue; + } + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + if( pTerm->wtFlags & TERM_LIKECOND ){ + /* If the TERM_LIKECOND flag is set, that means that the range search + ** is sufficient to guarantee that the LIKE operator is true, so we + ** can skip the call to the like(A,B) function. But this only works + ** for strings. So do not skip the call to the function on the pass + ** that compares BLOBs. */ #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - continue; + continue; #else - u32 x = pLevel->iLikeRepCntr; - assert( x>0 ); - skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1)); - VdbeCoverage(v); + u32 x = pLevel->iLikeRepCntr; + assert( x>0 ); + skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If, (int)(x>>1)); + VdbeCoverage(v); #endif + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); - pTerm->wtFlags |= TERM_CODED; + if( pIdx==0 ) break; + pIdx = 0; } /* Insert code to test for implied constraints based on transitivity diff --git a/test/pushdown.test b/test/pushdown.test new file mode 100644 index 0000000000..5be6054c08 --- /dev/null +++ b/test/pushdown.test @@ -0,0 +1,59 @@ +# 2017 April 29 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix pushdown + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 'b1', 'c1'); + INSERT INTO t1 VALUES(2, 'b2', 'c2'); + INSERT INTO t1 VALUES(3, 'b3', 'c3'); + INSERT INTO t1 VALUES(4, 'b4', 'c4'); + CREATE INDEX i1 ON t1(a, c); +} + +proc f {val} { + lappend ::L $val + return 0 +} +db func f f + +do_test 1.1 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=2 AND f(b) AND f(c) } + set L +} {c2} + +do_test 1.2 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=3 AND f(c) AND f(b) } + set L +} {c3} + +do_execsql_test 1.3 { + DROP INDEX i1; + CREATE INDEX i1 ON t1(a, b); +} +do_test 1.4 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=2 AND f(b) AND f(c) } + set L +} {b2} + +do_test 1.5 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=3 AND f(c) AND f(b) } + set L +} {b3} + +finish_test From 6ab3eb5d0eade5492508823c7f9405fc1a560987 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Apr 2017 14:56:55 +0000 Subject: [PATCH 2/2] Minor size and performance improvements to the push-down optimization. FossilOrigin-Name: 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/wherecode.c | 15 +++++++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 062de1d7ff..6cc1fccff6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Within\sa\sloop\sthat\suses\sa\snon-covering\sindex\stest,\stest\snon-indexed\sterms\sthat\ncan\sbe\stested\swithout\sseeking\sthe\smain\stable\scursor\sbefore\sthose\sthat\scannot. -D 2017-04-28T19:59:55.785 +C Minor\ssize\sand\sperformance\simprovements\sto\sthe\spush-down\soptimization. +D 2017-04-29T14:56:55.202 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -485,7 +485,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 -F src/wherecode.c 878a8c75af2bf6103ed4d1df639ee64ddb66f11b3a388f9caf7598c159a2c9af +F src/wherecode.c 8ad48867660519e262a401720845dc76934f86f558ec9606335fafcd7a2554f8 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1577,10 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 -R 8c650dff0f0f10ff63f37c52f3854c68 -T *branch * pushdown-optimization -T *sym-pushdown-optimization * -T -sym-trunk * -U dan -Z 464ba506caa7cc6d49be35980d40d709 +P afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 +R d3fbd82e46a4092d676255e41f20df5b +U drh +Z bf8e798698af6ed2b80cadbc03ebd2b9 diff --git a/manifest.uuid b/manifest.uuid index 0b47274c6c..2939f423e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 \ No newline at end of file +91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index a95889136a..feda45f911 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1130,6 +1130,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ Index *pIdx = 0; /* Index used by loop (if any) */ + int loopAgain; /* True if constraint generator loop should repeat */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -2027,8 +2028,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** This loop may run either once (pIdx==0) or twice (pIdx!=0). If ** it is run twice, then the first iteration codes those sub-expressions ** that can be computed using columns from pIdx only (without seeking - ** the main table cursor). */ - while( 1 ){ + ** the main table cursor). + */ + do{ + loopAgain = 0; for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; int skipLikeAddr = 0; @@ -2043,10 +2046,11 @@ Bitmask sqlite3WhereCodeOneLoopStart( } pE = pTerm->pExpr; assert( pE!=0 ); - if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ continue; } - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + loopAgain = 1; continue; } if( pTerm->wtFlags & TERM_LIKECOND ){ @@ -2068,9 +2072,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); pTerm->wtFlags |= TERM_CODED; } - if( pIdx==0 ) break; pIdx = 0; - } + }while( loopAgain ); /* Insert code to test for implied constraints based on transitivity ** of the "==" operator.