From 1b8fc65b92f4b0f982d4df5b1ea1659636adc060 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 7 Feb 2013 21:15:14 +0000 Subject: [PATCH 1/5] Allow an index to be used for sorting even if prior terms of the index are constrained by IN operators. FossilOrigin-Name: 98bf668ab1a8683b46ee8c94cb60f8215aabc517 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/where.c | 9 +++++++-- test/where.test | 28 ++++++++++++++++++++++++---- test/where2.test | 43 +++++++++++++++++++++++++++++++++++++++---- 5 files changed, 82 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 1df599f7db..8e947334a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-02-07T09:33:56.594 +C Allow\san\sindex\sto\sbe\sused\sfor\ssorting\seven\sif\sprior\sterms\sof\sthe\sindex\nare\sconstrained\sby\sIN\soperators. +D 2013-02-07T21:15:14.299 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 427c6ec07c2150e6d2bd28aefd98ab234cc374c3 +F src/where.c 74f8a582895bd27e74885858f598cdf54c32682d F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -970,8 +970,8 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 9714e6f292d70c22e78e1cecb3d896457186b9b7 -F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 +F test/where.test 55f6780f986b38b870e0fc50c3d842c13e7946a0 +F test/where2.test 61acb16d8e67e0d8717b2ad1a132f3b1ad81fd62 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1034,7 +1034,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e1640876df7ed6fb4e84292e5ce1f78635df64ab -R ed650c137d06abf5e97862ecc9965eee +P 4a7b4ee011fea911b981206c242e3d5553303b52 +R 5cc9a6226d6ff2cc56ed201a94d58505 +T *branch * IN-with-ORDERBY +T *sym-IN-with-ORDERBY * +T -sym-trunk * U drh -Z 0e5ff063c96a5d0e787292d2749e52de +Z 6d9a20d9ee8461cef5a45f6f7cc942b7 diff --git a/manifest.uuid b/manifest.uuid index 3c0ca4ea39..5c4fb5cc1c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a7b4ee011fea911b981206c242e3d5553303b52 \ No newline at end of file +98bf668ab1a8683b46ee8c94cb60f8215aabc517 \ No newline at end of file diff --git a/src/where.c b/src/where.c index f8da65af9a..99d00beb09 100644 --- a/src/where.c +++ b/src/where.c @@ -3010,10 +3010,15 @@ static int isSortingIndex( if( pConstraint==0 ){ isEq = 0; }else if( (pConstraint->eOperator & WO_IN)!=0 ){ +#if 0 /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY ** because we do not know in what order the values on the RHS of the IN ** operator will occur. */ break; +#else + if( termSortOrder ) break; + isEq = 0; +#endif }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ uniqueNotNull = 0; isEq = 1; /* "X IS NULL" means X has only a single value */ @@ -3317,8 +3322,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** indicate this to the caller. ** ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (pc.plan.nEq+1) that can be - ** optimized using the index. + ** there is a range constraint on indexed column (pc.plan.nEq+1) that + ** can be optimized using the index. */ if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){ testcase( pc.plan.wsFlags & WHERE_COLUMN_IN ); diff --git a/test/where.test b/test/where.test index 05c7b2f04f..4daa7fe34b 100644 --- a/test/where.test +++ b/test/where.test @@ -379,11 +379,26 @@ ifcapable subquery { SELECT * FROM t1 WHERE rowid+0 IN (1,2,3,1234) order by 1; } } {1 0 4 2 1 9 3 1 16 102} - do_test where-5.3 { + do_test where-5.3a { count { SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1; } - } {1 0 4 2 1 9 3 1 16 14} + } {1 0 4 2 1 9 3 1 16 13} + do_test where-5.3b { + count { + SELECT * FROM t1 WHERE w IN (3,-1,1,2) order by 1; + } + } {1 0 4 2 1 9 3 1 16 13} + do_test where-5.3c { + count { + SELECT * FROM t1 WHERE w IN (3,2,-1,1,2) order by 1; + } + } {1 0 4 2 1 9 3 1 16 13} + do_test where-5.3d { + count { + SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1 DESC; + } + } {3 1 16 2 1 9 1 0 4 14} do_test where-5.4 { count { SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1; @@ -511,11 +526,16 @@ do_test where-6.7 { } } {1 100 4 2 99 9 3 98 16 nosort} ifcapable subquery { - do_test where-6.8 { + do_test where-6.8a { cksort { SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a LIMIT 3 } - } {1 100 4 2 99 9 3 98 16 sort} + } {1 100 4 2 99 9 3 98 16 nosort} + do_test where-6.8b { + cksort { + SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a DESC LIMIT 3 + } + } {9 92 100 7 94 64 5 96 36 sort} } do_test where-6.9.1 { cksort { diff --git a/test/where2.test b/test/where2.test index d61c0897e8..7d71ffe254 100644 --- a/test/where2.test +++ b/test/where2.test @@ -167,24 +167,54 @@ ifcapable subquery { } } {99 6 10000 10006 100 6 10201 10207 sort t1 i1zyx} } - do_test where2-4.6 { + do_test where2-4.6a { queryplan { SELECT * FROM t1 WHERE x IN (1,2,3,4,5,6,7,8) AND y IN (10000,10001,10002,10003,10004,10005) - ORDER BY 2 + ORDER BY x + } + } {99 6 10000 10006 nosort t1 i1xy} + do_test where2-4.6b { + queryplan { + SELECT * FROM t1 + WHERE x IN (1,2,3,4,5,6,7,8) + AND y IN (10000,10001,10002,10003,10004,10005) + ORDER BY x DESC + } + } {99 6 10000 10006 sort t1 i1xy} + do_test where2-4.6c { + queryplan { + SELECT * FROM t1 + WHERE x IN (1,2,3,4,5,6,7,8) + AND y IN (10000,10001,10002,10003,10004,10005) + ORDER BY x, y + } + } {99 6 10000 10006 nosort t1 i1xy} + do_test where2-4.6d { + queryplan { + SELECT * FROM t1 + WHERE x IN (1,2,3,4,5,6,7,8) + AND y IN (10000,10001,10002,10003,10004,10005) + ORDER BY x, y DESC } } {99 6 10000 10006 sort t1 i1xy} # Duplicate entires on the RHS of an IN operator do not cause duplicate # output rows. # - do_test where2-4.6 { + do_test where2-4.6x { queryplan { SELECT * FROM t1 WHERE z IN (10207,10006,10006,10207) ORDER BY w } } {99 6 10000 10006 100 6 10201 10207 sort t1 i1zyx} + do_test where2-4.6y { + queryplan { + SELECT * FROM t1 WHERE z IN (10207,10006,10006,10207) + ORDER BY w DESC + } + } {100 6 10201 10207 99 6 10000 10006 sort t1 i1zyx} ifcapable compound { do_test where2-4.7 { queryplan { @@ -207,10 +237,15 @@ do_test where2-5.1 { } {99 6 10000 10006 nosort t1 i1w} ifcapable subquery { - do_test where2-5.2 { + do_test where2-5.2a { queryplan { SELECT * FROM t1 WHERE w IN (99) ORDER BY w } + } {99 6 10000 10006 nosort t1 i1w} + do_test where2-5.2b { + queryplan { + SELECT * FROM t1 WHERE w IN (99) ORDER BY w DESC + } } {99 6 10000 10006 sort t1 i1w} } From c3e552ff5cd7ec146769881ea773efcd10842f95 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Feb 2013 16:04:19 +0000 Subject: [PATCH 2/5] =?UTF-8?q?Allow=20the=20"a=3D=3F1=20OR=20a=3D=3F2"=20?= =?UTF-8?q?to=20"a=20IN=20(=3F1,=3F2)"=20transformation=20to=20work=20on?= =?UTF-8?q?=20virtual=20tables=20again.=20=20This=20was=20formerly=20restr?= =?UTF-8?q?icted=20because=20virtual=20tables=20could=20not=20optimize=20I?= =?UTF-8?q?N=20terms.=20=20(See=20check-in=20[fad88e71cf195e].)=20=20But?= =?UTF-8?q?=20IN=20terms=20are=20now=20used=20by=20virtual=20tables=20(as?= =?UTF-8?q?=20of=20check-in=20[3d65c70343])=20so=20the=20restriction=20can?= =?UTF-8?q?=20now=20be=20removed.?= FossilOrigin-Name: a917c1f09254b54e03e31b119cc49c551269d82e --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 17 ++--------------- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 8e947334a5..fd6cb8ae6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\san\sindex\sto\sbe\sused\sfor\ssorting\seven\sif\sprior\sterms\sof\sthe\sindex\nare\sconstrained\sby\sIN\soperators. -D 2013-02-07T21:15:14.299 +C Allow\sthe\s"a=?1\sOR\sa=?2"\sto\s"a\sIN\s(?1,?2)"\stransformation\sto\swork\son\svirtual\ntables\sagain.\s\sThis\swas\sformerly\srestricted\sbecause\svirtual\stables\scould\snot\noptimize\sIN\sterms.\s\s(See\scheck-in\s[fad88e71cf195e].)\s\sBut\sIN\sterms\sare\snow\nused\sby\svirtual\stables\s(as\sof\scheck-in\s[3d65c70343])\sso\sthe\srestriction\scan\nnow\sbe\sremoved. +D 2013-02-08T16:04:19.844 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 74f8a582895bd27e74885858f598cdf54c32682d +F src/where.c e09d21a4506a19c0a8b17e9614d362043c9ffeb1 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1034,10 +1034,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4a7b4ee011fea911b981206c242e3d5553303b52 -R 5cc9a6226d6ff2cc56ed201a94d58505 -T *branch * IN-with-ORDERBY -T *sym-IN-with-ORDERBY * -T -sym-trunk * +P 98bf668ab1a8683b46ee8c94cb60f8215aabc517 +R 77749b931237e0163d95186ceeef39fd U drh -Z 6d9a20d9ee8461cef5a45f6f7cc942b7 +Z afe286dca485d6e9ab8ba5a0ac8b0dbf diff --git a/manifest.uuid b/manifest.uuid index 5c4fb5cc1c..71e8e48900 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98bf668ab1a8683b46ee8c94cb60f8215aabc517 \ No newline at end of file +a917c1f09254b54e03e31b119cc49c551269d82e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 99d00beb09..14727270de 100644 --- a/src/where.c +++ b/src/where.c @@ -140,7 +140,6 @@ struct WhereTerm { struct WhereClause { Parse *pParse; /* The parser context */ WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ @@ -317,7 +316,6 @@ static void whereClauseInit( pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->vmask = 0; pWC->wctrlFlags = wctrlFlags; } @@ -917,7 +915,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** ** CASE 1: ** -** If all subterms are of the form T.C=expr for some single column of C +** If all subterms are of the form T.C=expr for some single column of C and ** a single table T (as shown in example B above) then create a new virtual ** term that is an equivalent IN expression. In other words, if the term ** being analyzed is: @@ -1005,7 +1003,7 @@ static void exprAnalyzeOrTerm( ** Compute the set of tables that might satisfy cases 1 or 2. */ indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); + chngToIN = ~(Bitmask)0; for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ WhereAndInfo *pAndInfo; @@ -5062,24 +5060,13 @@ WhereInfo *sqlite3WhereBegin( ** bitmask for all tables to the left of the join. Knowing the bitmask ** for all tables to the left of a left join is important. Ticket #3015. ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( sWBI.pWC->vmask==0 && pMaskSet->n==0 ); for(ii=0; iinSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[ii].pTab) && IsVirtual(pTabList->a[ii].pTab) ){ - sWBI.pWC->vmask |= ((Bitmask)1 << ii); - } -#endif } #ifndef NDEBUG { From 2d96b934c8da4141ef53e5a380467903e41c96e5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Feb 2013 18:48:23 +0000 Subject: [PATCH 3/5] Loop through the elements on the RHS of an IN operator in reverse order when the ORDER BY clauses specifies DESC. FossilOrigin-Name: f78395c8896666bb1359b83fbcd58d5e3dbc39d3 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 14 ++++---------- test/where.test | 28 ++++++++++++++++++++++++++-- test/where2.test | 4 ++-- 6 files changed, 43 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index fd6cb8ae6b..3637f1e541 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\s"a=?1\sOR\sa=?2"\sto\s"a\sIN\s(?1,?2)"\stransformation\sto\swork\son\svirtual\ntables\sagain.\s\sThis\swas\sformerly\srestricted\sbecause\svirtual\stables\scould\snot\noptimize\sIN\sterms.\s\s(See\scheck-in\s[fad88e71cf195e].)\s\sBut\sIN\sterms\sare\snow\nused\sby\svirtual\stables\s(as\sof\scheck-in\s[3d65c70343])\sso\sthe\srestriction\scan\nnow\sbe\sremoved. -D 2013-02-08T16:04:19.844 +C Loop\sthrough\sthe\selements\son\sthe\sRHS\sof\san\sIN\soperator\sin\sreverse\sorder\swhen\nthe\sORDER\sBY\sclauses\sspecifies\sDESC. +D 2013-02-08T18:48:23.501 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/shell.c 266791241d7add796ccce2317977ae6c3c67d77f F src/sqlite.h.in 39cc33bb08897c748fe3383c29ccf56585704177 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 6267485592261a1420ae9195e388242d9e451bdb +F src/sqliteInt.h c1e7fe135ec1957d4305f0f8016b5a12d339cd53 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c e09d21a4506a19c0a8b17e9614d362043c9ffeb1 +F src/where.c 36370580f2e3f805df161db44288bfb7d9ac007a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -970,8 +970,8 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 55f6780f986b38b870e0fc50c3d842c13e7946a0 -F test/where2.test 61acb16d8e67e0d8717b2ad1a132f3b1ad81fd62 +F test/where.test 15ac8611c9439a2c5f4a6ac10cfe4c1ec9854c24 +F test/where2.test 399b3178289925a0aa976b3d60ef139740540ecd F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1034,7 +1034,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 98bf668ab1a8683b46ee8c94cb60f8215aabc517 -R 77749b931237e0163d95186ceeef39fd +P a917c1f09254b54e03e31b119cc49c551269d82e +R f8ff7e2997f1e9085eb15a7df82507fc U drh -Z afe286dca485d6e9ab8ba5a0ac8b0dbf +Z d65a74f3ef1f6a5ee91ecd28380ca50d diff --git a/manifest.uuid b/manifest.uuid index 71e8e48900..8b749ce569 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a917c1f09254b54e03e31b119cc49c551269d82e \ No newline at end of file +f78395c8896666bb1359b83fbcd58d5e3dbc39d3 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f036970931..c58b0afb6e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1965,6 +1965,7 @@ struct WhereLevel { struct InLoop { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when plan.wsFlags&WHERE_IN_ABLE */ Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ diff --git a/src/where.c b/src/where.c index 14727270de..a3c5e24d39 100644 --- a/src/where.c +++ b/src/where.c @@ -3008,15 +3008,7 @@ static int isSortingIndex( if( pConstraint==0 ){ isEq = 0; }else if( (pConstraint->eOperator & WO_IN)!=0 ){ -#if 0 - /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY - ** because we do not know in what order the values on the RHS of the IN - ** operator will occur. */ - break; -#else - if( termSortOrder ) break; isEq = 0; -#endif }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ uniqueNotNull = 0; isEq = 1; /* "X IS NULL" means X has only a single value */ @@ -3786,12 +3778,13 @@ static int codeEqualityTerm( int eType; int iTab; struct InLoop *pIn; + u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; assert( pX->op==TK_IN ); iReg = iTarget; eType = sqlite3FindInIndex(pParse, pX, 0); iTab = pX->iTable; - sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); if( pLevel->u.in.nIn==0 ){ pLevel->addrNxt = sqlite3VdbeMakeLabel(v); @@ -3809,6 +3802,7 @@ static int codeEqualityTerm( }else{ pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; sqlite3VdbeAddOp1(v, OP_IsNull, iReg); }else{ pLevel->u.in.nIn = 0; @@ -5550,7 +5544,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } sqlite3DbFree(db, pLevel->u.in.aInLoop); diff --git a/test/where.test b/test/where.test index 4daa7fe34b..2dbc283417 100644 --- a/test/where.test +++ b/test/where.test @@ -398,7 +398,7 @@ ifcapable subquery { count { SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1 DESC; } - } {3 1 16 2 1 9 1 0 4 14} + } {3 1 16 2 1 9 1 0 4 12} do_test where-5.4 { count { SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1; @@ -467,6 +467,30 @@ ifcapable subquery { SELECT * FROM t1 WHERE x IN (1,7) AND y IN (9,16) ORDER BY 1; } } {2 1 9 3 1 16 11} + do_test where-5.100 { + db eval { + SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969) + ORDER BY x, y + } + } {2 1 9 54 5 3025 62 5 3969} + do_test where-5.101 { + db eval { + SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969) + ORDER BY x DESC, y DESC + } + } {62 5 3969 54 5 3025 2 1 9} + do_test where-5.102 { + db eval { + SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969) + ORDER BY x DESC, y + } + } {54 5 3025 62 5 3969 2 1 9} + do_test where-5.103 { + db eval { + SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969) + ORDER BY x, y DESC + } + } {2 1 9 62 5 3969 54 5 3025} } # This procedure executes the SQL. Then it checks to see if the OP_Sort @@ -535,7 +559,7 @@ ifcapable subquery { cksort { SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a DESC LIMIT 3 } - } {9 92 100 7 94 64 5 96 36 sort} + } {9 92 100 7 94 64 5 96 36 nosort} } do_test where-6.9.1 { cksort { diff --git a/test/where2.test b/test/where2.test index 7d71ffe254..e8c2f3613f 100644 --- a/test/where2.test +++ b/test/where2.test @@ -182,7 +182,7 @@ ifcapable subquery { AND y IN (10000,10001,10002,10003,10004,10005) ORDER BY x DESC } - } {99 6 10000 10006 sort t1 i1xy} + } {99 6 10000 10006 nosort t1 i1xy} do_test where2-4.6c { queryplan { SELECT * FROM t1 @@ -246,7 +246,7 @@ ifcapable subquery { queryplan { SELECT * FROM t1 WHERE w IN (99) ORDER BY w DESC } - } {99 6 10000 10006 sort t1 i1w} + } {99 6 10000 10006 nosort t1 i1w} } # Verify that OR clauses get translated into IN operators. From 3667033bfc236425644799261d461ddc3af909e3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Feb 2013 20:39:02 +0000 Subject: [PATCH 4/5] Make sure the virtual tables that take advantage of IN operators sort the RHS of the IN operator in the correct order according to the ORDER BY clause. FossilOrigin-Name: b016b7546d6fbfba06019398b9ac239b0cbe9086 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 27 +++++++++++++++++++-------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 3637f1e541..990aaddfef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Loop\sthrough\sthe\selements\son\sthe\sRHS\sof\san\sIN\soperator\sin\sreverse\sorder\swhen\nthe\sORDER\sBY\sclauses\sspecifies\sDESC. -D 2013-02-08T18:48:23.501 +C Make\ssure\sthe\svirtual\stables\sthat\stake\sadvantage\sof\sIN\soperators\ssort\sthe\nRHS\sof\sthe\sIN\soperator\sin\sthe\scorrect\sorder\saccording\sto\sthe\sORDER\sBY\nclause. +D 2013-02-08T20:39:02.391 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 36370580f2e3f805df161db44288bfb7d9ac007a +F src/where.c df3f8f610746165066bd029f24f35739b5d0a21f F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1034,7 +1034,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a917c1f09254b54e03e31b119cc49c551269d82e -R f8ff7e2997f1e9085eb15a7df82507fc +P f78395c8896666bb1359b83fbcd58d5e3dbc39d3 +R 080d4548a7ca8008fc94adcb8a7a49c2 U drh -Z d65a74f3ef1f6a5ee91ecd28380ca50d +Z 0d4d8d5d423c9c5e6216d31dab1445d7 diff --git a/manifest.uuid b/manifest.uuid index 8b749ce569..84f9305d63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f78395c8896666bb1359b83fbcd58d5e3dbc39d3 \ No newline at end of file +b016b7546d6fbfba06019398b9ac239b0cbe9086 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a3c5e24d39..c513c28b80 100644 --- a/src/where.c +++ b/src/where.c @@ -2270,8 +2270,9 @@ static void bestVirtualIndex(WhereBestIdx *p){ struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage; WhereTerm *pTerm; - int i, j; + int i, j, k; int nOrderBy; + int sortOrder; /* Sort order for IN clauses */ int bAllowIN; /* Allow IN optimizations */ double rCost; @@ -2370,18 +2371,27 @@ static void bestVirtualIndex(WhereBestIdx *p){ return; } + sortOrder = SQLITE_SO_ASC; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pIdxCons++){ if( pUsage[i].argvIndex>0 ){ j = pIdxCons->iTermOffset; pTerm = &pWC->a[j]; p->cost.used |= pTerm->prereqRight; - if( (pTerm->eOperator & WO_IN)!=0 && pUsage[i].omit==0 ){ - /* Do not attempt to use an IN constraint if the virtual table - ** says that the equivalent EQ constraint cannot be safely omitted. - ** If we do attempt to use such a constraint, some rows might be - ** repeated in the output. */ - break; + if( (pTerm->eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + for(k=0; knOrderBy; k++){ + if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){ + sortOrder = pIdxInfo->aOrderBy[k].desc; + break; + } + } } } } @@ -2411,7 +2421,8 @@ static void bestVirtualIndex(WhereBestIdx *p){ } p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - p->cost.plan.wsFlags |= WHERE_ORDERED; + assert( sortOrder==0 || sortOrder==1 ); + p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE; p->cost.plan.nOBSat = nOrderBy; }else{ p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; From 5edf8435d30c4049f84ed060081900cf23c2a598 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Feb 2013 23:18:18 +0000 Subject: [PATCH 5/5] Fix a potential NULL-pointer dereference following an OOM error in the query planner logic for virtual tables with OR-connected terms. FossilOrigin-Name: 71b6c26053fdf2a5a84116e005bad1f2ca873a66 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 990aaddfef..81f10ad5c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\svirtual\stables\sthat\stake\sadvantage\sof\sIN\soperators\ssort\sthe\nRHS\sof\sthe\sIN\soperator\sin\sthe\scorrect\sorder\saccording\sto\sthe\sORDER\sBY\nclause. -D 2013-02-08T20:39:02.391 +C Fix\sa\spotential\sNULL-pointer\sdereference\sfollowing\san\sOOM\serror\nin\sthe\squery\splanner\slogic\sfor\svirtual\stables\swith\sOR-connected\sterms. +D 2013-02-08T23:18:18.222 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c df3f8f610746165066bd029f24f35739b5d0a21f +F src/where.c 43e05406f0e05960a62d4719ed77f551f8204d3f F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1034,7 +1034,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f78395c8896666bb1359b83fbcd58d5e3dbc39d3 -R 080d4548a7ca8008fc94adcb8a7a49c2 +P b016b7546d6fbfba06019398b9ac239b0cbe9086 +R 754c451bf3b3a4bb7bca45cc0f3e7eb1 U drh -Z 0d4d8d5d423c9c5e6216d31dab1445d7 +Z bf21096dfb91901ebdb63502984eab24 diff --git a/manifest.uuid b/manifest.uuid index 84f9305d63..d0aef318b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b016b7546d6fbfba06019398b9ac239b0cbe9086 \ No newline at end of file +71b6c26053fdf2a5a84116e005bad1f2ca873a66 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c513c28b80..3db76b5339 100644 --- a/src/where.c +++ b/src/where.c @@ -3665,7 +3665,8 @@ static void bestIndex(WhereBestIdx *p){ sqlite3_index_info *pIdxInfo = 0; p->ppIdxInfo = &pIdxInfo; bestVirtualIndex(p); - if( pIdxInfo->needToFreeIdxStr ){ + assert( pIdxInfo!=0 || p->pParse->db->mallocFailed ); + if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){ sqlite3_free(pIdxInfo->idxStr); } sqlite3DbFree(p->pParse->db, pIdxInfo);