From 6457a353bca93d55db6011df34fb04e8c81e997e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Jun 2013 00:35:37 +0000 Subject: [PATCH 1/4] Modify the query planner interface so that it always passes in the result set. This is the first step toward adding an optimization that will omit tables from a join that do not contribute to the result. FossilOrigin-Name: 2c2577e69ccb47f1af674a755e71221e2ca0b322 --- manifest | 21 +++++++++++-------- manifest.uuid | 2 +- src/select.c | 7 ++++--- src/sqliteInt.h | 1 + src/where.c | 54 ++++++++++++++++++++++++++---------------------- test/alter2.test | 1 + 6 files changed, 48 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index e7e71b5de4..ede3df5706 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sNEVER()\smacro\sand\san\sexplanation\scomment\saround\san\sunreachable\sbranch\nin\sthe\sSTAT3\slogic. -D 2013-06-20T17:32:28.451 +C Modify\sthe\squery\splanner\sinterface\sso\sthat\sit\salways\spasses\sin\sthe\sresult\sset.\nThis\sis\sthe\sfirst\sstep\stoward\sadding\san\soptimization\sthat\swill\somit\stables\nfrom\sa\sjoin\sthat\sdo\snot\scontribute\sto\sthe\sresult. +D 2013-06-21T00:35:37.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,12 +215,12 @@ F src/printf.c bff529ed47657098c55c9910b9c69b1b3b1a1353 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c d5a1b9bc3fb451e68ce907df253c6ac17e7310f7 +F src/select.c 91b62654caf8dfe292fb8882715e575d34ad3874 F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 82a0f3e7c3410cc748c80268e0831dba5ea98cb4 +F src/sqliteInt.h cabeb0401566d80503a0bc6e2d12e7fe2577bf6d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,14 +289,14 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 8d6c07d9641bf107e03a2b613550d90b8c7f4a82 +F src/where.c c950b131584a40121092d735804472f567beefbc F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 57d96ec9b320bd07af77567034488dcb6642c748 -F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 +F test/alter2.test 40531b1f89d4fe43f9007b1bfc304e291ed000ae F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d F test/alter4.test b2debc14d8cbe4c1d12ccd6a41eef88a8c1f15d5 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc @@ -1096,7 +1096,10 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d94db3fd921890ab1d6414ab629410ae50779686 -R 4877ecf4e1d53d3b49ba9f1fadfb6d66 +P 604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 +R 54a7c9b08c105a408dfe2c4fd7fd4419 +T *branch * omit-join-table-opt +T *sym-omit-join-table-opt * +T -sym-nextgen-query-plan-exp * U drh -Z c939c499a0659dd37def1fcbcc79c922 +Z d2a3072c8b463b513f8068b32a4cb830 diff --git a/manifest.uuid b/manifest.uuid index bda9422793..169f9d22e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 \ No newline at end of file +2c2577e69ccb47f1af674a755e71221e2ca0b322 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 40b7ec86f0..fa35f45871 100644 --- a/src/select.c +++ b/src/select.c @@ -4261,15 +4261,16 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ - ExprList *pDist = (sDistinct.isTnct ? p->pEList : 0); + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList, + wctrlFlags, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); } - if( sqlite3WhereIsDistinct(pWInfo) ){ + if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); } if( pOrderBy && sqlite3WhereIsOrdered(pWInfo) ) pOrderBy = 0; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b2cab7a7ad..731612d345 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1974,6 +1974,7 @@ struct SrcList { #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ /* Allowed return values from sqlite3WhereIsDistinct() */ diff --git a/src/where.c b/src/where.c index cdc0e16839..ab4170fe6f 100644 --- a/src/where.c +++ b/src/where.c @@ -383,7 +383,7 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinct; /* DISTINCT ON values, or NULL */ + ExprList *pResultSet; /* Result set. DISTINCT operates on these */ WhereLoop *pLoops; /* List of all WhereLoop objects */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereCost nRowOut; /* Estimated number of output rows */ @@ -3928,9 +3928,9 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ int nb = 1+(pTabList->nSrc+7)/8; struct SrcList_item *pItem = pTabList->a + p->iTab; Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%c %2d.%0*llx.%0*llx", p->cId, + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq); - sqlite3DebugPrintf(" %8s", + sqlite3DebugPrintf(" %12s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ if( p->u.btree.pIndex ){ @@ -3941,9 +3941,9 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ while( zName[i]!='_' ) i--; zName += i; } - sqlite3DebugPrintf(".%-12s %2d", zName, p->u.btree.nEq); + sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); }else{ - sqlite3DebugPrintf("%16s",""); + sqlite3DebugPrintf("%20s",""); } }else{ char *z; @@ -3953,10 +3953,10 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ }else{ z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); } - sqlite3DebugPrintf(" %-15s", z); + sqlite3DebugPrintf(" %-19s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" fg %05x N %d", p->wsFlags, p->nLTerm); + sqlite3DebugPrintf(" f %04x N %d", p->wsFlags, p->nLTerm); sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); } #endif @@ -5370,12 +5370,12 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pLevel->iFrom = pWLoop->iTab; pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; } - if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 - && pWInfo->pDistinct + if( (pWInfo->wctrlFlags & (WHERE_DISTINCTBY|WHERE_WANT_DISTINCT)) + ==WHERE_WANT_DISTINCT && nRowEst ){ Bitmask notUsed; - int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinct, pFrom, + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } @@ -5464,7 +5464,9 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; if( pWInfo->pOrderBy ) pWInfo->bOBSat = 1; - if( pWInfo->pDistinct ) pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } #ifdef SQLITE_DEBUG pLoop->cId = '0'; #endif @@ -5554,10 +5556,10 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ */ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ - SrcList *pTabList, /* A list of all tables to be scanned */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ - ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ + ExprList *pResultSet, /* Result set of the query */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ ){ @@ -5613,7 +5615,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; - pWInfo->pDistinct = pDistinct; + pWInfo->pResultSet = pResultSet; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; @@ -5628,7 +5630,9 @@ WhereInfo *sqlite3WhereBegin( /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( OptimizationDisabled(db, SQLITE_DistinctOpt) ) pDistinct = 0; + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. @@ -5650,7 +5654,9 @@ WhereInfo *sqlite3WhereBegin( */ if( nTabList==0 ){ if( pOrderBy ) pWInfo->bOBSat = 1; - if( pDistinct ) pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } } /* Assign a bit from the bitmask to every term in the FROM clause. @@ -5697,7 +5703,7 @@ WhereInfo *sqlite3WhereBegin( ** expressions, then we won't be able to satisfy it using indices, so ** go ahead and disable it now. */ - if( pOrderBy && pDistinct ){ + if( pOrderBy && (wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOrderBy->a[ii].pExpr); if( pExpr->op!=TK_COLUMN ){ @@ -5709,17 +5715,15 @@ WhereInfo *sqlite3WhereBegin( } } - /* Check if the DISTINCT qualifier, if there is one, is redundant. - ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to - ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. - */ - if( pDistinct ){ - if( isDistinctRedundant(pParse,pTabList,&pWInfo->sWC,pDistinct) ){ - pDistinct = 0; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + /* The DISTINCT marking is pointless. Ignore it. */ + wctrlFlags &= ~WHERE_WANT_DISTINCT; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ + /* Try to ORDER BY the result set to make distinct processing easier */ pWInfo->wctrlFlags |= WHERE_DISTINCTBY; - pWInfo->pOrderBy = pDistinct; + pWInfo->pOrderBy = pResultSet; } } diff --git a/test/alter2.test b/test/alter2.test index 14be637f97..db8a83bf9f 100644 --- a/test/alter2.test +++ b/test/alter2.test @@ -120,6 +120,7 @@ do_test alter2-1.5 { } } {} do_test alter2-1.6 { +breakpoint execsql { SELECT c FROM abc ORDER BY c; } From fd636c7541d0de19ce365fc9e483b88121531c2e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Jun 2013 02:05:06 +0000 Subject: [PATCH 2/4] Attempt to disable inner loops of a join that do not generate output. This does not work, since the inner loops might run zero times and thus inhibit all output. Needs to be enhanced to work only for LEFT JOINs or when we know that the inner loop will always run at least once. FossilOrigin-Name: ca839723a21bb13d3e0666a672c15c6f3a267c2f --- manifest | 17 +++++++--------- manifest.uuid | 2 +- src/where.c | 51 +++++++++++++++++++++++++++++++++--------------- test/alter2.test | 1 - 4 files changed, 43 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index ede3df5706..2d7ec778f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\squery\splanner\sinterface\sso\sthat\sit\salways\spasses\sin\sthe\sresult\sset.\nThis\sis\sthe\sfirst\sstep\stoward\sadding\san\soptimization\sthat\swill\somit\stables\nfrom\sa\sjoin\sthat\sdo\snot\scontribute\sto\sthe\sresult. -D 2013-06-21T00:35:37.456 +C Attempt\sto\sdisable\sinner\sloops\sof\sa\sjoin\sthat\sdo\snot\sgenerate\soutput.\nThis\sdoes\snot\swork,\ssince\sthe\sinner\sloops\smight\srun\szero\stimes\sand\sthus\ninhibit\sall\soutput.\s\sNeeds\sto\sbe\senhanced\sto\swork\sonly\sfor\sLEFT\sJOINs\nor\swhen\swe\sknow\sthat\sthe\sinner\sloop\swill\salways\srun\sat\sleast\sonce. +D 2013-06-21T02:05:06.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,14 +289,14 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c c950b131584a40121092d735804472f567beefbc +F src/where.c fc5293b54a70474c2b46e9df26c9e2803b152e68 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 57d96ec9b320bd07af77567034488dcb6642c748 -F test/alter2.test 40531b1f89d4fe43f9007b1bfc304e291ed000ae +F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d F test/alter4.test b2debc14d8cbe4c1d12ccd6a41eef88a8c1f15d5 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc @@ -1096,10 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 604c3c5de6fd8f8a569aa9ed981055a5b0123ba1 -R 54a7c9b08c105a408dfe2c4fd7fd4419 -T *branch * omit-join-table-opt -T *sym-omit-join-table-opt * -T -sym-nextgen-query-plan-exp * +P 2c2577e69ccb47f1af674a755e71221e2ca0b322 +R b2263c66dd8a59884ee45351c335261a U drh -Z d2a3072c8b463b513f8068b32a4cb830 +Z 6fd83e64fe937f64082b799e88969f81 diff --git a/manifest.uuid b/manifest.uuid index 169f9d22e6..b8cca3c3d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c2577e69ccb47f1af674a755e71221e2ca0b322 \ No newline at end of file +ca839723a21bb13d3e0666a672c15c6f3a267c2f \ No newline at end of file diff --git a/src/where.c b/src/where.c index ab4170fe6f..18b4a3782e 100644 --- a/src/where.c +++ b/src/where.c @@ -392,10 +392,10 @@ struct WhereInfo { u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + u8 nLevel; /* Number of nested loop */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ - int nLevel; /* Number of nested loop */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereClause sWC; /* Decomposition of the WHERE clause */ @@ -5370,8 +5370,9 @@ static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ pLevel->iFrom = pWLoop->iTab; pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; } - if( (pWInfo->wctrlFlags & (WHERE_DISTINCTBY|WHERE_WANT_DISTINCT)) - ==WHERE_WANT_DISTINCT + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->eDistinct==WHERE_DISTINCT_NOOP && nRowEst ){ Bitmask notUsed; @@ -5571,15 +5572,23 @@ WhereInfo *sqlite3WhereBegin( WhereLoopBuilder sWLB; /* The WhereLoop builder */ WhereMaskSet *pMaskSet; /* The expression mask set */ WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ int ii; /* Loop counter */ sqlite3 *db; /* Database connection */ int rc; /* Return code */ /* Variable initialization */ + db = pParse->db; memset(&sWLB, 0, sizeof(sWLB)); sWLB.pOrderBy = pOrderBy; + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } + /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ @@ -5603,7 +5612,6 @@ WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - db = pParse->db; nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ @@ -5628,12 +5636,6 @@ WhereInfo *sqlite3WhereBegin( sWLB.pNew->cId = '*'; #endif - /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via - ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ - wctrlFlags &= ~WHERE_WANT_DISTINCT; - } - /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ @@ -5718,7 +5720,6 @@ WhereInfo *sqlite3WhereBegin( if( wctrlFlags & WHERE_WANT_DISTINCT ){ if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ - wctrlFlags &= ~WHERE_WANT_DISTINCT; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ /* Try to ORDER BY the result set to make distinct processing easier */ @@ -5737,11 +5738,11 @@ WhereInfo *sqlite3WhereBegin( #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ WhereLoop *p; - int i = 0; + int i; static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; - for(p=pWInfo->pLoops; p; p=p->pNextLoop){ - p->cId = zLabel[(i++)%sizeof(zLabel)]; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%sizeof(zLabel)]; whereLoopPrint(p, pTabList); } } @@ -5782,11 +5783,29 @@ WhereInfo *sqlite3WhereBegin( } } sqlite3DebugPrintf("\n"); - for(ii=0; iinLevel; ii++){ whereLoopPrint(pWInfo->a[ii].pWLoop, pTabList); } } #endif + /* Attempt to omit tables from the join that do not effect the result */ + if( pResultSet!=0 && pWInfo->nLevel>=2 ){ + Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); + if( pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, pOrderBy); + while( pWInfo->nLevel>=2 ){ + pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; + if( ((wctrlFlags & WHERE_WANT_DISTINCT)!=0 + || (pLoop->wsFlags & WHERE_ONEROW)!=0) + && (tabUsed & pLoop->maskSelf)==0 + ){ + WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId)); + pWInfo->nLevel--; + nTabList--; + }else{ + break; + } + } + } WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; @@ -5955,7 +5974,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ /* Close all of the cursors that were opened by sqlite3WhereBegin. */ - assert( pWInfo->nLevel==1 || pWInfo->nLevel==pTabList->nSrc ); + assert( pWInfo->nLevel<=pTabList->nSrc ); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ Index *pIdx = 0; struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; diff --git a/test/alter2.test b/test/alter2.test index db8a83bf9f..14be637f97 100644 --- a/test/alter2.test +++ b/test/alter2.test @@ -120,7 +120,6 @@ do_test alter2-1.5 { } } {} do_test alter2-1.6 { -breakpoint execsql { SELECT c FROM abc ORDER BY c; } From bc71b1d4536ce7842f80010049c69f5fd2806b3b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Jun 2013 02:15:48 +0000 Subject: [PATCH 3/4] Only eliminate inner loops of a JOIN if they are the RHS of a LEFT JOIN and if they give no more than a single result. This appears to give correct answers in all cases. FossilOrigin-Name: d7a25cc79794817504ca1a4262008a68b2a4dece --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 2d7ec778f6..a54bc73242 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sdisable\sinner\sloops\sof\sa\sjoin\sthat\sdo\snot\sgenerate\soutput.\nThis\sdoes\snot\swork,\ssince\sthe\sinner\sloops\smight\srun\szero\stimes\sand\sthus\ninhibit\sall\soutput.\s\sNeeds\sto\sbe\senhanced\sto\swork\sonly\sfor\sLEFT\sJOINs\nor\swhen\swe\sknow\sthat\sthe\sinner\sloop\swill\salways\srun\sat\sleast\sonce. -D 2013-06-21T02:05:06.206 +C Only\seliminate\sinner\sloops\sof\sa\sJOIN\sif\sthey\sare\sthe\sRHS\sof\sa\sLEFT\sJOIN\nand\sif\sthey\sgive\sno\smore\sthan\sa\ssingle\sresult.\s\sThis\sappears\sto\sgive\scorrect\nanswers\sin\sall\scases. +D 2013-06-21T02:15:48.306 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c fc5293b54a70474c2b46e9df26c9e2803b152e68 +F src/where.c 0ca9544cca6e40550b970574a32055463650df12 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2c2577e69ccb47f1af674a755e71221e2ca0b322 -R b2263c66dd8a59884ee45351c335261a +P ca839723a21bb13d3e0666a672c15c6f3a267c2f +R 6b62f4d6f88cb9d978862e981acfa473 U drh -Z 6fd83e64fe937f64082b799e88969f81 +Z b99e6e4ea8d5a45c0b5323f96846f0b0 diff --git a/manifest.uuid b/manifest.uuid index b8cca3c3d8..32c308b877 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca839723a21bb13d3e0666a672c15c6f3a267c2f \ No newline at end of file +d7a25cc79794817504ca1a4262008a68b2a4dece \ No newline at end of file diff --git a/src/where.c b/src/where.c index 18b4a3782e..32fa61c7f0 100644 --- a/src/where.c +++ b/src/where.c @@ -5794,16 +5794,16 @@ WhereInfo *sqlite3WhereBegin( if( pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, pOrderBy); while( pWInfo->nLevel>=2 ){ pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( ((wctrlFlags & WHERE_WANT_DISTINCT)!=0 - || (pLoop->wsFlags & WHERE_ONEROW)!=0) - && (tabUsed & pLoop->maskSelf)==0 + if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ - WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId)); - pWInfo->nLevel--; - nTabList--; - }else{ break; } + if( (tabUsed & pLoop->maskSelf)!=0 ) break; + WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId)); + pWInfo->nLevel--; + nTabList--; } } WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); From 1031bd99581c1c3d7da99eb1c9a74a4cf053e0da Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Jun 2013 15:44:26 +0000 Subject: [PATCH 4/4] Add the ability to disable the omit-join-table optimization for testing purposes. FossilOrigin-Name: d929df9b1ba214c27d8c437099a53ee9b07aa169 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 5 ++++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a54bc73242..1bb860ddcd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\seliminate\sinner\sloops\sof\sa\sJOIN\sif\sthey\sare\sthe\sRHS\sof\sa\sLEFT\sJOIN\nand\sif\sthey\sgive\sno\smore\sthan\sa\ssingle\sresult.\s\sThis\sappears\sto\sgive\scorrect\nanswers\sin\sall\scases. -D 2013-06-21T02:15:48.306 +C Add\sthe\sability\sto\sdisable\sthe\somit-join-table\soptimization\sfor\stesting\npurposes. +D 2013-06-22T15:44:26.893 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/shell.c ab6eea968c8745be3aa74e45fedb37d057b4cd0d F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h cabeb0401566d80503a0bc6e2d12e7fe2577bf6d +F src/sqliteInt.h e6f069b07fdef1ab54034940b7a6e7be2b4efd57 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0ca9544cca6e40550b970574a32055463650df12 +F src/where.c 9bcfcb4ec6a14dd0111bf287bee02be88d5709f9 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1096,7 +1096,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ca839723a21bb13d3e0666a672c15c6f3a267c2f -R 6b62f4d6f88cb9d978862e981acfa473 +P d7a25cc79794817504ca1a4262008a68b2a4dece +R ca6216e1e7f1e0a7cea14bc87e274a42 U drh -Z b99e6e4ea8d5a45c0b5323f96846f0b0 +Z 6a72a8bdf1b2e20868f42b8b65540410 diff --git a/manifest.uuid b/manifest.uuid index 32c308b877..f7dd8d6565 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7a25cc79794817504ca1a4262008a68b2a4dece \ No newline at end of file +d929df9b1ba214c27d8c437099a53ee9b07aa169 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 731612d345..0a661937ff 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1024,6 +1024,7 @@ struct sqlite3 { #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/where.c b/src/where.c index 32fa61c7f0..e20f2b3018 100644 --- a/src/where.c +++ b/src/where.c @@ -5789,7 +5789,10 @@ WhereInfo *sqlite3WhereBegin( } #endif /* Attempt to omit tables from the join that do not effect the result */ - if( pResultSet!=0 && pWInfo->nLevel>=2 ){ + if( pWInfo->nLevel>=2 + && pResultSet!=0 + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + ){ Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); if( pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, pOrderBy); while( pWInfo->nLevel>=2 ){