From 648fe49f96dc485e9ab0d2a7b6c4c0e69cdaac85 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Sep 2017 20:06:53 +0000 Subject: [PATCH 1/5] Do not flatten subqueries that contain an ORDER BY or GROUP BY clause and can be implemented using a co-routine. FossilOrigin-Name: 042d655dd9002e8b89a798ad955b0285891aecf79f6978c5312e70ffe0609a46 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/select.c | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d11bdc4fb7..c63192e1bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\s(undocumented)\squery-planner\scontrol\sthat\sprevents\s\na\s"SELECT\sALL"\ssubquery\sin\sFROM\sclause\sfrom\sbeing\simplemented\sas\sa\sco-routine. -D 2017-09-28T17:29:24.426 +C Do\snot\sflatten\ssubqueries\sthat\scontain\san\sORDER\sBY\sor\sGROUP\sBY\sclause\sand\ncan\sbe\simplemented\susing\sa\sco-routine. +D 2017-09-28T20:06:53.014 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c ff9551b1792d838d1973ca3f5062804e2a00b1fa2cef6d7931992d89f2b3744a +F src/select.c d38163a76a0da425afd4a43f5dd0dc88fdbe0aa258c45f6a9cb44acf0a606bc5 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 @@ -1655,7 +1655,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fd3267ef92384fcefaee7460a5ffbaf8ddcb6049eec36f72a7046a43e2871fbf -R 5a9f08df25678ade0635d27ff9994d5e +P ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18 +R 2327508642786ef6d3dd975eaebc22c4 +T *branch * prefer-coroutine-sort-subquery +T *sym-prefer-coroutine-sort-subquery * +T -sym-trunk * U drh -Z 15e07447876306f708cc7a309f77f755 +Z 34c1093fc7e9ecc2e1ce3f4d0b9d734a diff --git a/manifest.uuid b/manifest.uuid index 6515252213..61c790dd19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff2f5a31a2ac67a2fdbb25793e8013cb0e062ab90bdcba2d52a62d6d4d8b6d18 \ No newline at end of file +042d655dd9002e8b89a798ad955b0285891aecf79f6978c5312e70ffe0609a46 \ No newline at end of file diff --git a/src/select.c b/src/select.c index a5f04767bf..7583fc24f1 100644 --- a/src/select.c +++ b/src/select.c @@ -5205,6 +5205,25 @@ int sqlite3Select( goto select_end; } + /* If the subquery contains an ORDER BY or GROUP BY clause and if + ** it will be implemented as a co-routine, then do not flatten. This + ** restriction allows SQL constructs like this: + ** + ** SELECT expensive_function(x) + ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + ** + ** The expensive_function() is only computed on the 10 rows that + ** are output, rather than every row of the table. + */ + if( (pSub->pOrderBy!=0 || pSub->pGroupBy!=0) + && i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + continue; + } + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ /* This subquery can be absorbed into its parent. */ From 25c221eb1dcc72ac0e524c605d61042ae91a6fe8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Sep 2017 22:13:24 +0000 Subject: [PATCH 2/5] Always render a subquery that is not part of a join as a co-routine. FossilOrigin-Name: 6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37 --- manifest | 16 ++++----- manifest.uuid | 2 +- src/select.c | 94 +++++++++++++++++++------------------------------ src/sqliteInt.h | 18 +++++----- src/test1.c | 1 - 5 files changed, 53 insertions(+), 78 deletions(-) diff --git a/manifest b/manifest index 26986bd8f3..35a58bfa19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\squery\sflattener\scomment\simprovements\sfrom\strunk. -D 2017-09-29T16:08:46.956 +C Always\srender\sa\ssubquery\sthat\sis\snot\spart\sof\sa\sjoin\sas\sa\sco-routine. +D 2017-09-29T22:13:24.684 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -458,18 +458,18 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 73fa1cc03a325dd5f3e84bc6c4c975000028f96f85025ae2fe0b6eb0c30c2fe0 +F src/select.c 8e8e9965af11f7ca54a44fc2cea9e8d162a6ffa1ccd2e840c69ff4468e0d08e0 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 -F src/sqliteInt.h 954fed875c59d283870f13010bbbb96d8b25cf3db0683843403c36e9dfcf1bdd +F src/sqliteInt.h d51b1b549b6f64a44b0f5f5620aeb5bba414cf07704978238885b273ce2aa84f F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 487951d81f9704800fd9f0ffdaa2f935a83ccb6be3575c2c4ef83e4789b4c828 -F src/test1.c a947b2554fa77d0ef2dd21d1ef08e37e5d91b17af83de923a4e3c7f10957a2eb +F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06f432fb7c979f1bb7f01f5c90716ce5c0248f73f70b78a9870b9de5c9bf7ef4 0840f9f824c16212ce3fd6c859e501176eb0a58924ea1728a54d5bdfd0c25c86 -R 4be3453c15fd85db566735e226eb00f9 +P f62cd4d940506c39db82e83ff3df8ab1856f1fb91ffda835ae2d727263ee9b0b +R 5c67a1a4877d581279b4ed945a250449 U drh -Z be149bf809ee0d7792ea4e04a36ef7f0 +Z 6d48696ae0bb75b33e85df88734665c6 diff --git a/manifest.uuid b/manifest.uuid index bef919b1e8..143260d747 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f62cd4d940506c39db82e83ff3df8ab1856f1fb91ffda835ae2d727263ee9b0b \ No newline at end of file +6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37 \ No newline at end of file diff --git a/src/select.c b/src/select.c index adc23d887a..0f46d73f31 100644 --- a/src/select.c +++ b/src/select.c @@ -3298,9 +3298,11 @@ static void substSelect( ** ** Flattening is subject to the following constraints: ** -** (1) The subquery and the outer query cannot both be aggregates. +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery and the outer query cannot both be aggregates. ** -** (2) If the subquery is an aggregate then +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** (2) If the subquery is an aggregate then ** (2a) the outer query must not be a join and ** (2b) the outer query must not use subqueries ** other than the one FROM-clause subquery that is a candidate @@ -3319,7 +3321,8 @@ static void substSelect( ** sub-queries that were excluded from this optimization. Restriction ** (4) has since been expanded to exclude all DISTINCT subqueries. ** -** (6) If the subquery is aggregate, the outer query may not be DISTINCT. +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** If the subquery is aggregate, the outer query may not be DISTINCT. ** ** (7) The subquery must have a FROM clause. TODO: For subqueries without ** A FROM clause, consider adding a FROM clause with the special @@ -3406,7 +3409,7 @@ static void substSelect( ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query -** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. +** uses aggregates. ** ** If flattening is not attempted, this routine is a no-op and returns 0. ** If flattening is attempted this routine returns 1. @@ -3418,8 +3421,7 @@ static int flattenSubquery( Parse *pParse, /* Parsing context */ Select *p, /* The parent or outer SELECT statement */ int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ - int isAgg, /* True if outer SELECT uses aggregate functions */ - int subqueryIsAgg /* True if the subquery uses aggregate functions */ + int isAgg /* True if outer SELECT uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; Select *pParent; /* Current UNION ALL term of the other query */ @@ -3446,16 +3448,6 @@ static int flattenSubquery( iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); - if( subqueryIsAgg ){ - if( isAgg ) return 0; /* Restriction (1) */ - if( pSrc->nSrc>1 ) return 0; /* Restriction (2a) */ - if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery)) - || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0 - || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 - ){ - return 0; /* Restriction (2b) */ - } - } pSubSrc = pSub->pSrc; assert( pSubSrc ); @@ -3474,9 +3466,6 @@ static int flattenSubquery( if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ return 0; /* Restrictions (8)(9) */ } - if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ - return 0; /* Restriction (6) */ - } if( p->pOrderBy && pSub->pOrderBy ){ return 0; /* Restriction (11) */ } @@ -3778,18 +3767,7 @@ static int flattenSubquery( if( isLeftJoin>0 ){ setJoinExpr(pWhere, iNewParent); } - if( subqueryIsAgg ){ - assert( pParent->pHaving==0 ); - pParent->pHaving = pParent->pWhere; - pParent->pWhere = pWhere; - pParent->pHaving = sqlite3ExprAnd(db, - sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving - ); - assert( pParent->pGroupBy==0 ); - pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); - }else{ - pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); - } + pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); if( db->mallocFailed==0 ){ SubstContext x; x.pParse = pParse; @@ -3852,9 +3830,9 @@ static int flattenSubquery( ** ** Do not attempt this optimization if: ** -** (1) The inner query is an aggregate. (In that case, we'd really want -** to copy the outer WHERE-clause terms onto the HAVING clause of the -** inner query. But they probably won't help there so do not bother.) +** (1) (** This restriction was removed on 2017-09-29. We used to +** disallow this optimization for aggregate subqueries, but now +** it is allowed by putting the extra terms on the HAVING clause **) ** ** (2) The inner query is the recursive part of a common table expression. ** @@ -3882,11 +3860,9 @@ static int pushDownWhereTerms( Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; for(pX=pSubq; pX; pX=pX->pPrior){ - if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - testcase( pX->selFlags & SF_Aggregate ); - testcase( pX->selFlags & SF_Recursive ); + if( (pX->selFlags & (SF_Recursive))!=0 ){ testcase( pX!=pSubq ); - return 0; /* restrictions (1) and (2) */ + return 0; /* restriction (2) */ } } if( pSubq->pLimit!=0 ){ @@ -3896,7 +3872,7 @@ static int pushDownWhereTerms( nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor); pWhere = pWhere->pLeft; } - if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction (5) */ if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ @@ -3908,7 +3884,11 @@ static int pushDownWhereTerms( x.isLeftJoin = 0; x.pEList = pSubq->pEList; pNew = substExpr(&x, pNew); - pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); + if( pSubq->selFlags & SF_Aggregate ){ + pSubq->pHaving = sqlite3ExprAnd(pParse->db, pSubq->pHaving, pNew); + }else{ + pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); + } pSubq = pSubq->pPrior; } } @@ -5202,7 +5182,6 @@ int sqlite3Select( for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; Select *pSub = pItem->pSelect; - int isAggSub; Table *pTab = pItem->pTab; if( pSub==0 ) continue; @@ -5214,7 +5193,17 @@ int sqlite3Select( goto select_end; } - /* If the subquery contains an ORDER BY or GROUP BY clause and if + /* Do not try to flatten an aggregate subquery. + ** + ** Flattening an aggregate subquery is only possible if the outer query + ** is not a join. But if the outer query is not a join, then the subquery + ** will be implemented as a co-routine and there is no advantage to + ** flattening in that case. + */ + if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; + assert( pSub->pGroupBy==0 ); + + /* If the subquery contains an ORDER BY clause and if ** it will be implemented as a co-routine, then do not flatten. This ** restriction allows SQL constructs like this: ** @@ -5224,22 +5213,16 @@ int sqlite3Select( ** The expensive_function() is only computed on the 10 rows that ** are output, rather than every row of the table. */ - if( (pSub->pOrderBy!=0 || pSub->pGroupBy!=0) + if( pSub->pOrderBy!=0 && i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) - && OptimizationEnabled(db, SQLITE_SubqCoroutine) ){ continue; } - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + if( flattenSubquery(pParse, p, i, isAgg) ){ /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; - } i = -1; } pTabList = p->pSrc; @@ -5348,12 +5331,9 @@ int sqlite3Select( /* Generate code to implement the subquery ** - ** The subquery is implemented as a co-routine if all of these are true: - ** (1) The subquery is guaranteed to be the outer loop (so that it - ** does not need to be computed more than once) - ** (2) REMOVED (2017-09-28): The ALL keyword after SELECT is omitted. - ** (3) Co-routines are not disabled using sqlite3_test_control() - ** with SQLITE_TESTCTRL_OPTIMIZATIONS. + ** The subquery is implemented as a co-routine if the subquery is + ** guaranteed to be the outer loop (so that it does not need to be + ** computed more than once) ** ** TODO: Are there other reasons beside (1) to use a co-routine ** implementation? @@ -5361,8 +5341,6 @@ int sqlite3Select( if( i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - /*** constraint removed: && (p->selFlags & SF_All)==0 (2) */ - && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3259df894f..9b45295e70 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1506,16 +1506,14 @@ struct sqlite3 { #define SQLITE_ColumnCache 0x0002 /* Column cache */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ -/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ -#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ -#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ -#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ -#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ -#define SQLITE_Transitive 0x0200 /* Transitive constraints */ -#define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ -#define SQLITE_CountOfView 0x1000 /* The count-of-view optimization */ -#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ +#define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x0020 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x0040 /* ORDER BY of joins via index */ +#define SQLITE_Transitive 0x0080 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ +#define SQLITE_Stat34 0x0200 /* Use STAT3 or STAT4 data */ +#define SQLITE_CountOfView 0x0400 /* The count-of-view optimization */ +#define SQLITE_CursorHints 0x0800 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/test1.c b/src/test1.c index d70ce77a7a..446317d803 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6901,7 +6901,6 @@ static int SQLITE_TCLAPI optimization_control( { "cover-idx-scan", SQLITE_CoverIdxScan }, { "order-by-idx-join", SQLITE_OrderByIdxJoin }, { "transitive", SQLITE_Transitive }, - { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, { "stat3", SQLITE_Stat34 }, { "stat4", SQLITE_Stat34 }, From 508e2d00f34f2a4d8a04130de6bf2cf44eeb1921 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 30 Sep 2017 01:25:04 +0000 Subject: [PATCH 3/5] Fix unreachable conditionals and revise a testcase that was made obsolete by the changes on this branch. FossilOrigin-Name: 71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 11 +++++------ src/select.c | 27 +++++++++++++++++---------- test/having.test | 31 ++++++++++++++++++------------- 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 35a58bfa19..8d9fc53ff7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\srender\sa\ssubquery\sthat\sis\snot\spart\sof\sa\sjoin\sas\sa\sco-routine. -D 2017-09-29T22:13:24.684 +C Fix\sunreachable\sconditionals\sand\srevise\sa\stestcase\sthat\swas\smade\sobsolete\sby\nthe\schanges\son\sthis\sbranch. +D 2017-09-30T01:25:04.328 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -411,7 +411,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 -F src/expr.c 628395aea8bbf4e2851565f9dab76565e3100ca0836a337fb748834c9cec9f04 +F src/expr.c 4d2d0aafd945424f638ee03e11330f03288ccf616e025498f3c8602d01609a0a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 5ff2c895fe087756d8085dc1a9bc229b5670e2a65c3929dd87c71e43649af333 F src/func.c b4d259af627e3cd9510cf08db37f0bcc88b1887c735169c74490c3739d5cf5c6 @@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 8e8e9965af11f7ca54a44fc2cea9e8d162a6ffa1ccd2e840c69ff4468e0d08e0 +F src/select.c 55a17d54eec9093baf4ccc25342e4b2168cb8198805ed08a44fd4ae389b34b19 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 @@ -911,7 +911,7 @@ F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/having.test b3d6b17cc9601b6b373b2d0f08c075ccf30e2d307249c3c8a236e3c36907b1a5 +F test/having.test e4098a4b8962f9596035c3b87a8928a10648acc509f1bb8d6f96413bbf79a1b3 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 @@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f62cd4d940506c39db82e83ff3df8ab1856f1fb91ffda835ae2d727263ee9b0b -R 5c67a1a4877d581279b4ed945a250449 +P 6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37 +R 3a66fd0c62efe88b91fd1bdc729b86a5 U drh -Z 6d48696ae0bb75b33e85df88734665c6 +Z 6d29940c14d977f820858db31a47d732 diff --git a/manifest.uuid b/manifest.uuid index 143260d747..2da525fc6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37 \ No newline at end of file +71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 8282c99a4a..0b922cd0fd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1712,12 +1712,11 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ u32 sqlite3ExprListFlags(const ExprList *pList){ int i; u32 m = 0; - if( pList ){ - for(i=0; inExpr; i++){ - Expr *pExpr = pList->a[i].pExpr; - assert( pExpr!=0 ); - m |= pExpr->flags; - } + assert( pList!=0 ); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + m |= pExpr->flags; } return m; } diff --git a/src/select.c b/src/select.c index 0f46d73f31..9c3544ae05 100644 --- a/src/select.c +++ b/src/select.c @@ -3401,7 +3401,8 @@ static void substSelect( ** parent to a compound query confuses the code that handles ** recursive queries in multiSelect(). ** -** (24) The subquery may not be an aggregate that uses the built-in min() or +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery may not be an aggregate that uses the built-in min() or ** or max() functions. (Without this restriction, a query like: ** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily ** return the value X for which Y was maximal.) @@ -3474,10 +3475,8 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } - testcase( pSub->selFlags & SF_Recursive ); - testcase( pSub->selFlags & SF_MinMaxAgg ); - if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){ - return 0; /* Restrictions (22) and (24) */ + if( pSub->selFlags & (SF_Recursive) ){ + return 0; /* Restrictions (22) */ } if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ return 0; /* Restriction (23) */ @@ -3857,14 +3856,22 @@ static int pushDownWhereTerms( ){ Expr *pNew; int nChng = 0; - Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; - for(pX=pSubq; pX; pX=pX->pPrior){ - if( (pX->selFlags & (SF_Recursive))!=0 ){ - testcase( pX!=pSubq ); - return 0; /* restriction (2) */ + if( pSubq->selFlags & SF_Recursive ) return 0; /* restriction (2) */ + +#ifdef SQLITE_DEBUG + /* Only the first term of a compound can have a WITH clause. But make + ** sure no other terms are marked SF_Recursive in case something changes + ** in the future. + */ + { + Select *pX; + for(pX=pSubq; pX; pX=pX->pPrior){ + assert( (pX->selFlags & (SF_Recursive))==0 ); } } +#endif + if( pSubq->pLimit!=0 ){ return 0; /* restriction (3) */ } diff --git a/test/having.test b/test/having.test index aea12319d7..a3882552d3 100644 --- a/test/having.test +++ b/test/having.test @@ -65,19 +65,6 @@ foreach {tn sql1 sql2} { 3 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE binary HAVING a=2" "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a COLLATE binary" - 4 { - SELECT x,y FROM ( - SELECT a AS x, sum(b) AS y FROM t1 - GROUP BY a - ) WHERE x BETWEEN 8888 AND 9999 - } { - SELECT x,y FROM ( - SELECT a AS x, sum(b) AS y FROM t1 - WHERE x BETWEEN 8888 AND 9999 - GROUP BY a - ) - } - 5 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE binary HAVING 0" "SELECT a, sum(b) FROM t1 WHERE 0 GROUP BY a COLLATE binary" @@ -98,6 +85,24 @@ foreach {tn sql1 sql2} { do_compare_vdbe_test 2.$tn $sql1 $sql2 1 } +# The (4) test in the above set used to generate identical bytecode, but +# that is no longer the case. The byte code is equivalent, though. +# +do_execsql_test 2.4a { + SELECT x,y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 + GROUP BY a + ) WHERE x BETWEEN 2 AND 9999 +} {2 12} +do_execsql_test 2.4b { + SELECT x,y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 + WHERE x BETWEEN 2 AND 9999 + GROUP BY a + ) +} {2 12} + + #------------------------------------------------------------------------- # 1: Test that the optimization is only applied if the GROUP BY term # uses BINARY collation. From 24ae373ab1c3e484c3013048a215a48d72a103de Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 30 Sep 2017 10:50:34 +0000 Subject: [PATCH 4/5] Make sure the SQLITE_Stat34 optimization switch is always 0x800, a value which is hard-coded in the TH3 test suite. FossilOrigin-Name: 6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8d9fc53ff7..010e54d3d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sunreachable\sconditionals\sand\srevise\sa\stestcase\sthat\swas\smade\sobsolete\sby\nthe\schanges\son\sthis\sbranch. -D 2017-09-30T01:25:04.328 +C Make\ssure\sthe\sSQLITE_Stat34\soptimization\sswitch\sis\salways\s0x800,\sa\svalue\nwhich\sis\shard-coded\sin\sthe\sTH3\stest\ssuite. +D 2017-09-30T10:50:34.309 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -464,7 +464,7 @@ F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220e F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a1fd3aa82f967da436164e0728a7d6841651fd0c6e27b9044e0eb9f6c8462e47 -F src/sqliteInt.h d51b1b549b6f64a44b0f5f5620aeb5bba414cf07704978238885b273ce2aa84f +F src/sqliteInt.h 99e4beebd466495434ca6ed94bd7966a7f8c5879cd1f01768c588a5a2acd84c2 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b1651d711eae6e7c65a191f02ca2439160bcd677099712289e76a0f8422fd37 -R 3a66fd0c62efe88b91fd1bdc729b86a5 +P 71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e +R abcf7527b871a7332be788888884086b U drh -Z 6d29940c14d977f820858db31a47d732 +Z a8cf074d964a4716a4bd6f57e83e01aa diff --git a/manifest.uuid b/manifest.uuid index 2da525fc6b..6801513137 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e \ No newline at end of file +6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9b45295e70..99c8b73748 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1511,9 +1511,10 @@ struct sqlite3 { #define SQLITE_OrderByIdxJoin 0x0040 /* ORDER BY of joins via index */ #define SQLITE_Transitive 0x0080 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ -#define SQLITE_Stat34 0x0200 /* Use STAT3 or STAT4 data */ -#define SQLITE_CountOfView 0x0400 /* The count-of-view optimization */ -#define SQLITE_CursorHints 0x0800 /* Add OP_CursorHint opcodes */ +#define SQLITE_CountOfView 0x0200 /* The count-of-view optimization */ +#define SQLITE_CursorHints 0x0400 /* Add OP_CursorHint opcodes */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ + /* TH3 expects the Stat34 ^^^^^^ value to be 0x0800. Don't change it */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* From 67cc51a4919ae33125715e4742ce7a1ada1ad8ac Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 30 Sep 2017 11:47:06 +0000 Subject: [PATCH 5/5] More details in the header comment of pushDownWhereTerms(). No changes to code. FossilOrigin-Name: 928486231ff7cc0df17a5aa23a080a1b8720e168db5227a7dbd34861283ba689 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 010e54d3d6..55d7ee5a1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sSQLITE_Stat34\soptimization\sswitch\sis\salways\s0x800,\sa\svalue\nwhich\sis\shard-coded\sin\sthe\sTH3\stest\ssuite. -D 2017-09-30T10:50:34.309 +C More\sdetails\sin\sthe\sheader\scomment\sof\spushDownWhereTerms().\s\sNo\schanges\sto\ncode. +D 2017-09-30T11:47:06.786 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -458,7 +458,7 @@ F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 55a17d54eec9093baf4ccc25342e4b2168cb8198805ed08a44fd4ae389b34b19 +F src/select.c 19ebf10e33de8e1f327cbc844aff12af74bdf0513b3eb1b712b7bcdca4c9bdb5 F src/shell.c c1206a23d9239f8f51751d3be9b8c3b02fa4103546bea1add7f864d84a8276ab F src/shell.c.in bb9720a8c5c98d3984b16ab7540e7142bcae959666ecf248bfc523a1d44220ee F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 @@ -1655,7 +1655,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 71f0adf7ca6824c3aba69104b9976dbb71b377474529e1a36220b4804293501e -R abcf7527b871a7332be788888884086b +P 6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc +R f601c290e8e6d8b405b953711d4d99d6 U drh -Z a8cf074d964a4716a4bd6f57e83e01aa +Z 5fa1972988b6fbd3340440a07005926a diff --git a/manifest.uuid b/manifest.uuid index 6801513137..8a76d04632 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6aed4ea34c4163c682ad5bb2956fdf4f3a3ad048fefd3edab0fef6761c3783cc \ No newline at end of file +928486231ff7cc0df17a5aa23a080a1b8720e168db5227a7dbd34861283ba689 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9c3544ae05..b7c459011c 100644 --- a/src/select.c +++ b/src/select.c @@ -3831,7 +3831,11 @@ static int flattenSubquery( ** ** (1) (** This restriction was removed on 2017-09-29. We used to ** disallow this optimization for aggregate subqueries, but now -** it is allowed by putting the extra terms on the HAVING clause **) +** it is allowed by putting the extra terms on the HAVING clause. +** The added HAVING clause is pointless if the subquery lacks +** a GROUP BY clause. But such a HAVING clause is also harmless +** so there does not appear to be any reason to add extra logic +** to suppress it. **) ** ** (2) The inner query is the recursive part of a common table expression. **