diff --git a/manifest b/manifest index cead004f71..70d136a298 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smemory\sleaks\sin\sthe\sNGQP\slogic\sfor\svirtual\stables. -D 2013-05-08T20:05:58.332 +C Free\sup\sbits\sof\swsFlags\sfor\sreuse.\s\sInstall\sthe\sORDER\sBY\soptimization\ninfrastructure\sfor\sthe\sNGQP. +D 2013-05-10T02:00:35.314 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h aee09cf5ee7d5e899246affdd8c3cf58f086ade5 +F src/sqliteInt.h b4411cfc1c7803cdf393975d5420b4da0d8dd1c4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e2e0ff816591684ff74bef65fae747cc57cee335 +F src/where.c 2be67c1a1a018b1de91f08e48baa02db13652e40 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bd9327a9684b99978734ccd561eea1ad864ab13b -R 08bf850cd4f323af143e8919e5b89b44 +P 3c2e83a4a2c5e85202162feeb37ef7a3911c05a3 +R fe9c54bd236af22a2f9e27cec8708db5 U drh -Z 89b960e01fe84155abc15edf55ac2ba0 +Z c9a2e102e5146d5a293863d252ad4424 diff --git a/manifest.uuid b/manifest.uuid index 6cc7d9d5c8..5c1c65ebf9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c2e83a4a2c5e85202162feeb37ef7a3911c05a3 \ No newline at end of file +82d50e198025a2fdb8ee733edb8419d388ee5362 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 86b123749c..08ecfa78d1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2045,6 +2045,7 @@ struct WhereLevel { #define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ +#define WHREE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ /* ** The WHERE clause processing routine has two halves. The @@ -2056,6 +2057,8 @@ struct WhereLevel { struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pDistinct; /* DISTINCT ON values, or NULL */ u16 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ diff --git a/src/where.c b/src/where.c index 6d7805dfdd..56ca3e75aa 100644 --- a/src/where.c +++ b/src/where.c @@ -50,7 +50,7 @@ typedef struct WhereVtabPlan WhereVtabPlan; /* ** Each instance of this object represents a way of evaluating one ** term of a join. The WhereClause object holds a table of these -** objects using (iTab,prereq,iOb,nOb) as the primary key. Note that the +** objects using (maskSelf,prereq,) as the primary key. Note that the ** same join term might have multiple associated WhereLoop objects. */ struct WhereLoop { @@ -58,19 +58,19 @@ struct WhereLoop { Bitmask maskSelf; /* Bitmask identifying table iTab */ u16 iTab; /* Index of the table coded by this loop */ u16 nTerm; /* Number of entries in aTerm[] */ - u16 iOb, nOb; /* ORDER BY terms satisfied by this strategy */ + u32 wsFlags; /* WHERE_* flags describing the plan */ double rSetup; /* One-time setup cost (ex: create transient index) */ double rRun; /* Cost of running each loop */ double nOut; /* Estimated number of output rows */ - u32 wsFlags; /* WHERE_* flags describing the plan */ union { struct { /* Information for internal btree tables */ int nEq; /* Number of equality constraints */ Index *pIndex; /* Index used, or NULL */ } btree; - struct { /* Information for virtualt tables */ + struct { /* Information for virtual tables */ int idxNum; /* Index number */ - int needFree; /* True if sqlite3_free(idxStr) is needed */ + u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + u8 isOrdered; /* True if satisfies ORDER BY */ char *idxStr; /* Index identifier string */ } vtab; } u; @@ -86,6 +86,8 @@ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ double nRow; /* Estimated number of rows generated by this path */ double rCost; /* Total cost of this path */ + u8 isOrdered; /* True if this path satisfies ORDER BY */ + u8 isOrderedValid; /* True if the isOrdered field is valid */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; @@ -314,41 +316,34 @@ struct WhereLoopBuilder { ** Value for wsFlags returned by bestIndex() and stored in ** WhereLevel.wsFlags. These flags determine which search ** strategies are appropriate. -** -** The least significant 12 bits is reserved as a mask for WO_ values above. -** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL. -** But if the table is the right table of a left join, WhereLevel.wsFlags -** is set to WO_IN|WO_EQ. The WhereLevel.wsFlags field can then be used as -** the "op" parameter to findTerm when we are resolving equality constraints. -** ISNULL constraints will then not be used on the right table of a left -** join. Tickets #2177 and #2189. */ -#define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ -#define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ -#define WHERE_IPK 0x00008000 /* x is the INTEGER PRIMARY KEY */ -#define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ -#define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ -#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x080f1000 /* Able to support an IN operator */ -#define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xEXPR */ +#define WHERE_NULL_OK 0x00000004 /* Ok to use WO_ISNULL */ +#define WHERE_IPK 0x00000008 /* x is the INTEGER PRIMARY KEY */ +#define WHERE_COLUMN_EQ 0x00000010 /* x=EXPR or x IN (...) or x IS NULL */ +#define WHERE_COLUMN_RANGE 0x00000020 /* xEXPR */ +#define WHERE_COLUMN_IN 0x00000040 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000080 /* x IS NULL */ +#define WHERE_INDEXED 0x000000f0 /* Anything that uses an index */ +#define WHERE_NOT_FULLSCAN 0x000200f3 /* Does not do a full table scan */ +#define WHERE_IN_ABLE 0x000100f1 /* Able to support an IN operator */ +#define WHERE_TOP_LIMIT 0x00000100 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000300 /* Both x>EXPR and xu.pOrInfo->wc; WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; WhereTerm *pOrTerm; - int flags = WHERE_MULTI_OR; double rTotal = 0; double nRow = 0; Bitmask used = 0; @@ -1967,7 +1961,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ p->cost.used = used; p->cost.plan.nRow = nRow; p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; - p->cost.plan.wsFlags = flags; + p->cost.plan.wsFlags = WHERE_MULTI_OR; p->cost.plan.u.pTerm = pTerm; } } @@ -2149,7 +2143,7 @@ static void constructAutomaticIndex( if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ nColumn += pTable->nCol - BMS + 1; } - pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ; + pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ nByte = sizeof(Index); @@ -3815,7 +3809,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ bestOrClauseIndex(p); bestAutomaticIndex(p); - p->cost.plan.wsFlags |= eqTermMask; + if( eqTermMask & WO_ISNULL ) p->cost.plan.wsFlags |= WHERE_NULL_OK; } /* @@ -4066,6 +4060,7 @@ static int codeAllEqualityTerms( int regBase; /* Base register */ int nReg; /* Number of registers to allocate */ char *zAff; /* Affinity string to return */ + int eqFlags; /* WO_EQ|WO_IN and maybe also WO_ISNULL */ /* This module is only called on query plans that use an index. */ assert( pLevel->plan.wsFlags & WHERE_INDEXED ); @@ -4085,10 +4080,12 @@ static int codeAllEqualityTerms( /* Evaluate the equality constraints */ assert( pIdx->nColumn>=nEq ); + eqFlags = (pLevel->plan.wsFlags&WHERE_NULL_OK) ? (WO_EQ|WO_IN|WO_ISNULL) + : (WO_EQ|WO_IN); for(j=0; jaiColumn[j]; - pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); + pTerm = findTerm(pWC, iCur, k, notReady, eqFlags, pIdx); if( pTerm==0 ) break; /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ @@ -5090,8 +5087,7 @@ static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ sqlite3DebugPrintf(" %-15s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", - p->wsFlags, p->iOb, p->nOb, p->nTerm); + sqlite3DebugPrintf(" fg %08x N %2d", p->wsFlags, p->nTerm); sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", p->prereq, p->rSetup, p->rRun, p->nOut); } @@ -5172,8 +5168,6 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->prereq & pTemplate->prereq)==p->prereq - && p->nOb>=pTemplate->nOb - && p->iOb==pTemplate->iOb && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun ){ @@ -5182,8 +5176,6 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ return SQLITE_OK; } if( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->nOb<=pTemplate->nOb - && p->iOb==pTemplate->iOb && p->rSetup>=pTemplate->rSetup && p->rRun>=pTemplate->rRun ){ @@ -5364,7 +5356,6 @@ static int whereLoopAddBtree( pNew->prereq = 0; pNew->u.btree.pIndex = 0; pNew->wsFlags = 0; - pNew->iOb = pNew->nOb = 0; pNew->rRun = (double)pSrc->pTab->nRowEst; pNew->nOut = (double)pSrc->pTab->nRowEst; rc = whereLoopInsert(pBuilder->pWInfo, pNew); @@ -5451,8 +5442,6 @@ static int whereLoopAddVirtual( pNew->prereq = 0; pNew->iTab = iTab; pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, pSrc->iCursor); - pNew->iOb = 0; - pNew->nOb = 0; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; pNew->nTerm = 0; @@ -5547,12 +5536,7 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->iOb = 0; - if( pIdxInfo->orderByConsumed ){ - pNew->nOb = (u16)(pIdxInfo->nOrderBy&0xffff); - }else{ - pNew->nOb = 0; - } + pNew->u.vtab.isOrdered = (u8)(pIdxInfo->nOrderBy!=0); pNew->rSetup = (double)0; pNew->rRun = pIdxInfo->estimatedCost; pNew->nOut = (double)25; @@ -5604,6 +5588,29 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ return rc; } +/* +** Examine a WherePath to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate source operation. Return 1 +** if it does and 0 if it does not and -1 if we cannot tell. +*/ +static int wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + WherePath *pPath, /* The WherePath to check */ + int nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLoop /* Add this WhereLoop to the end of pPath->aLoop[] */ +){ + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + return nLoop==0 && pLoop->u.vtab.isOrdered; + }else{ + /* TBD: Check to see if pFrom + pWLoop satisfies the ORDER BY. + ** (1) If yes: set isOrderedValid and isOrdered to 1. + ** (2) If no: set isOrderedValid to 1 and isOrdered to 0. + ** (3) unknown: no-op */ + return 0; + } +} + + /* ** Given the list of WhereLoop objects on pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -5620,12 +5627,14 @@ static int wherePathSolver(WhereInfo *pWInfo){ int ii, jj; /* Loop counters */ double rCost; /* Cost of a path */ double mxCost; /* Maximum cost of a set of paths */ + double rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ WherePath *pFrom; /* An element of aFrom[] that we are working on */ WherePath *pTo; /* An element of aTo[] that we are working on */ WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop *pNext; /* Next loop */ WhereLoop **pX; /* Used to divy up the pSpace memory */ char *pSpace; /* Temporary memory used by this routine */ @@ -5645,18 +5654,67 @@ static int wherePathSolver(WhereInfo *pWInfo){ pFrom->aLoop = pX; } + /* Seed the search with a single WherePath containing zero WhereLoops */ aFrom[0].nRow = (double)1; nFrom = 1; + + /* Precompute the cost of sorting the final result set, if the caller + ** to sqlite3WhereBegin() was concerned about sorting */ + rSortCost = (double)0; + if( pWInfo->pOrderBy==0 ){ + aFrom[0].isOrderedValid = 1; + }else{ + /* Compute an estimate on the cost to sort the entire result set */ + rSortCost = (double)1; + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pNext){ + pNext = pWLoop->pNextLoop; + rCost = pWLoop->nOut; + while( pNext && pNext->iTab==pWLoop->iTab ){ + if( pNext->nOutnOut; + pNext = pNext->pNextLoop; + } + rSortCost *= rCost; + } + rSortCost *= estLog(rSortCost); + } + + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ Bitmask maskNew; + u8 isOrderedValid = pFrom->isOrderedValid; + u8 isOrdered = pFrom->isOrdered; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ rCost = pWLoop->rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; maskNew = pFrom->maskLoop | pWLoop->maskSelf; - for(jj=0, pTo=aTo; jjmaskLoop!=maskNew; jj++){} + if( !isOrderedValid ){ + switch( wherePathSatisfiesOrderBy(pWInfo, pFrom, iLoop, pWLoop) ){ + case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ + isOrdered = 1; + isOrderedValid = 1; + break; + case 0: /* No. pFrom+pWLoop will require a separate sort */ + isOrdered = 0; + isOrderedValid = 1; + rCost += rSortCost; + break; + default: /* Cannot tell yet. Try again on the next iteration */ + break; + } + } + /* Check to see if pWLoop should be added to the mxChoice best so far */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew && pTo->isOrderedValid==isOrderedValid ){ + break; + } + } if( jj>=nTo ){ if( nTo>=mxChoice && rCost>=mxCost ) continue; if( nTorCost<=rCost ) continue; } + /* pWLoop is a winner. Add it to the set of best so far */ pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; pTo->nRow = pFrom->nRow * pWLoop->nOut; pTo->rCost = rCost; + pTo->isOrderedValid = isOrderedValid; + pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; if( nTo>=mxChoice ){ @@ -5695,8 +5756,7 @@ static int wherePathSolver(WhereInfo *pWInfo){ } #endif - /* Swap the roles of aFrom and aTo in preparation for the next - ** cycle. */ + /* Swap the roles of aFrom and aTo for the next generation */ pFrom = aTo; aTo = aFrom; aFrom = pFrom; @@ -5707,15 +5767,19 @@ static int wherePathSolver(WhereInfo *pWInfo){ if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } assert( nFrom>0 ); - /* Find the lowest cost path and load it into pWInfo->a[].pWLoop */ + /* Find the lowest cost path. pFrom will be left pointing to that path */ pFrom = aFrom; for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; } assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLoopa[iLoop].pWLoop = pFrom->aLoop[iLoop]; } + if( pFrom->isOrdered ){ + pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + } /* Free temporary memory and return success */ sqlite3DbFree(db, pSpace); @@ -5882,6 +5946,8 @@ WhereInfo *sqlite3WhereBegin( pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pDistinct = pDistinct; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); pWInfo->pWC = sWBI.pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags;