diff --git a/manifest b/manifest index 91047af181..c355e4ffe5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Combine\sthe\sFuncDef.iPrefEnc\sand\sFuncDef.flags\sfields\sinto\sa\ssingle\nnew\sFuncDef.funcFlags\sfield. -D 2013-09-06T13:10:12.385 +C Initial\simplementation\sof\sthe\sunlikely()\sSQL\sfunction\sused\sas\sa\shint\sto\nthe\squery\splanner. +D 2013-09-06T15:23:29.191 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,10 +172,10 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 2dc64ca360b7d7da481183ea920a813a0c203c97 -F src/expr.c 33ae44812d25799000dbcfd54bb2142b30cdd049 +F src/expr.c d0ed048b3b3c97fddbcccb8df43cc39b89e682c2 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c be866cd8c4fa6cae98ba33109578fd1a3311ee5b -F src/func.c 7650d35651bb0ca903d24f4e5e944bfdaac9f152 +F src/func.c a297b1f3cf48dd483f5453b7155bc5e8f4ca1fc2 F src/global.c 5caf4deab621abb45b4c607aad1bd21c20aac759 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -214,14 +214,14 @@ F src/pragma.c 3aa3d8c8623b7b71c5b1bfb72dcc31fb0c25665f F src/prepare.c fa6988589f39af8504a61731614cd4f6ae71554f F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 -F src/resolve.c 9d53899cc6e1f4ec0b4632d07e97d57827bf63b9 +F src/resolve.c 748618c3df7f37f2cd33c8834770b1e34f9b1185 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 9239586282bd146ec5843a2cde7d54cd7816cf78 F src/shell.c d920a891ca09b8bd262cced7fb0ab9d723f7a747 F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 32e959dba9899360ce4ae3cd6cc77b4a8df277e8 +F src/sqliteInt.h fd9005fab1cd063f5c4e76a2120efc080b43750a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -290,7 +290,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 06c249a5137575ecf4d527e5a1dadb1087e4375c +F src/where.c c9f8ae51797bebfbfa49b9e5bede6fc142b44262 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 4a48461d71a7b4a958570c580495a97e842e2700 +F test/analyze9.test e688c7289d3d33b7185e2c2afcac6a9f095563a3 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1111,7 +1111,10 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a99a53b81e29c9514b85318bea028d1667e5f760 -R 8c10a6011eb52c6098e833ca515e6c77 +P 97b10e66e98e84755aa577f8da017bf1aea2056c +R fcd7cd29a90a12a870cace66939b8877 +T *branch * unlikely-func +T *sym-unlikely-func * +T -sym-trunk * U drh -Z c7e49c79214a66374f550dcdb7a70012 +Z 2a30cee668099b5c8e0e2b447ac67feb diff --git a/manifest.uuid b/manifest.uuid index dbc304a491..b43b68fee1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97b10e66e98e84755aa577f8da017bf1aea2056c \ No newline at end of file +036fc37a034093a4c6fc190633bd41c2b7230d77 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 6587ee1f78..ece0293840 100644 --- a/src/expr.c +++ b/src/expr.c @@ -89,8 +89,16 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ ** an expression. */ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ - while( pExpr && (pExpr->op==TK_COLLATE || pExpr->op==TK_AS) ){ - pExpr = pExpr->pLeft; + while( pExpr ){ + if( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ){ + pExpr = pExpr->pLeft; + }else if( ExprHasAnyProperty(pExpr, EP_Hint) ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( pExpr->x.pList->nExpr>0 ); + pExpr = pExpr->x.pList->a[0].pExpr; + }else{ + break; + } } return pExpr; } @@ -2649,6 +2657,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + assert( nFarg>=1 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + break; + } if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); diff --git a/src/func.c b/src/func.c index 64e33640b0..26cbfbfb03 100644 --- a/src/func.c +++ b/src/func.c @@ -418,14 +418,14 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } /* -** The COALESCE() and IFNULL() functions are implemented as VDBE code so -** that unused argument values do not have to be computed. However, we -** still need some kind of function implementation for this routines in -** the function table. That function implementation will never be called -** so it doesn't matter what the implementation is. We might as well use -** the "version()" function as a substitute. +** Some functions like COALESCE() and IFNULL() and UNLIKELY() are implemented +** as VDBE code so that unused argument values do not have to be computed. +** However, we still need some kind of function implementation for this +** routines in the function table. The noopFunc macro provides this. +** noopFunc will never be called so it doesn't matter what the implementation +** is. We might as well use the "version()" function as a substitute. */ -#define ifnullFunc versionFunc /* Substitute function - never called */ +#define noopFunc versionFunc /* Substitute function - never called */ /* ** Implementation of random(). Return a random integer. @@ -1659,9 +1659,11 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), - FUNCTION2(coalesce, -1, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), - FUNCTION2(ifnull, 2, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), + FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(unlikely, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/src/resolve.c b/src/resolve.c index 43a3870e23..cb86a83ee0 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -570,6 +570,18 @@ static void notValidCheckConstraint( # define notValidCheckConstraint(P,N,M) #endif +/* +** Expression p should encode a floating point value between 1.0 and 0.0. +** Return 1024 times this value. Or return -1 if p is not a floating point +** value between 1.0 and 0.0. +*/ +static int exprProbability(Expr *p){ + double r = -1.0; + if( p->op!=TK_FLOAT ) return -1; + sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + if( r<0.0 || r>1.0 ) return -1; + return (int)(r*1000.0); +} /* ** This routine is callback for sqlite3WalkExpr(). @@ -683,6 +695,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } }else{ is_agg = pDef->xFunc==0; + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + ExprSetProperty(pExpr, EP_Hint); + if( n==2 ){ + pExpr->iTable = exprProbability(pList->a[1].pExpr); + if( pExpr->iTable<0 ){ + sqlite3ErrorMsg(pParse, "second parameter to unlikely() must be " + "between 0.0 and 1.0"); + pNC->nErr++; + } + }else{ + pExpr->iTable = 100; + } + } } #ifndef SQLITE_OMIT_AUTHORIZATION if( pDef ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6b3fbe8518..d8ed4eb04e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1752,7 +1752,8 @@ struct Expr { #endif int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number - ** TK_TRIGGER: 1 -> new, 0 -> old */ + ** TK_TRIGGER: 1 -> new, 0 -> old + ** EP_Hint: 1000 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ @@ -1780,7 +1781,7 @@ struct Expr { #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Hint 0x1000 /* Not used */ +#define EP_Hint 0x1000 /* The UNLIKELY() SQL function */ #define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ #define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ #define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ diff --git a/src/where.c b/src/where.c index 3c8a1eaac6..a9905769a4 100644 --- a/src/where.c +++ b/src/where.c @@ -52,12 +52,13 @@ typedef struct WhereOrSet WhereOrSet; ** Cost X is tracked as 10*log2(X) stored in a 16-bit integer. The ** maximum cost for ordinary tables is 64*(2**63) which becomes 6900. ** (Virtual tables can return a larger cost, but let's assume they do not.) -** So all costs can be stored in a 16-bit unsigned integer without risk +** So all costs can be stored in a 16-bit integer without risk ** of overflow. ** ** Costs are estimates, so no effort is made to compute 10*log2(X) exactly. -** Instead, a close estimate is used. Any value of X<=1 is stored as 0. -** X=2 is 10. X=3 is 16. X=1000 is 99. etc. +** Instead, a close estimate is used. Any value of X=1 is stored as 0. +** X=2 is 10. X=3 is 16. X=1000 is 99. etc. Negative values are allowed. +** A WhereCost of -10 means 0.5. WhereCost of -20 means 0.25. And so forth. ** ** The tool/wherecosttest.c source file implements a command-line program ** that will convert WhereCosts to integers, convert integers to WhereCosts @@ -65,7 +66,7 @@ typedef struct WhereOrSet WhereOrSet; ** command-line program is a useful utility to have around when working with ** this module. */ -typedef unsigned short int WhereCost; +typedef short int WhereCost; /* ** This object contains information needed to implement a single nested @@ -268,6 +269,7 @@ struct WhereTerm { WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; + WhereCost truthProb; /* Probability of truth for this expression */ u16 eOperator; /* A WO_xx value describing */ u8 wtFlags; /* TERM_xxx bit flags. See below */ u8 nChild; /* Number of children that must disable us */ @@ -642,6 +644,9 @@ static void whereClauseClear(WhereClause *pWC){ } } +/* Forward declaration */ +static WhereCost whereCost(tRowcnt x); + /* ** Add a single new WhereTerm entry to the WhereClause object pWC. ** The new WhereTerm object is constructed from Expr p and with wtFlags. @@ -683,6 +688,11 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); } pTerm = &pWC->a[idx = pWC->nTerm++]; + if( p && ExprHasAnyProperty(p, EP_Hint) ){ + pTerm->truthProb = whereCost(p->iTable) - 99; + }else{ + pTerm->truthProb = -1; + } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; @@ -4257,6 +4267,32 @@ whereLoopInsert_noop: return SQLITE_OK; } +/* +** Adjust the WhereLoop.nOut value downward to account for terms of the +** WHERE clause that reference the loop but which are not used by an +** index. +** +** In the current implementation, the first extra WHERE clause term reduces +** the number of output rows by a factor of 10 and each additional term +** reduces the number of output rows by sqrt(2). +*/ +static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop, int iCur){ + WhereTerm *pTerm; + Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); + int x = 0; + int i; + for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + x += pTerm->truthProb; + } + for(i=pLoop->nLTerm-1; i>=0; i--){ + x -= pLoop->aLTerm[i]->truthProb; + } + if( x<0 ) pLoop->nOut += x; +} + /* ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. ** Try to match one more. @@ -4423,7 +4459,7 @@ static int whereLoopAddBtreeIndex( } /* Step cost for each output row */ pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut); - /* TBD: Adjust nOut for additional constraints */ + whereLoopOutputAdjust(pBuilder->pWC, pNew, pSrc->iCursor); rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nColumn + (pProbe->zName!=0)) @@ -4627,7 +4663,9 @@ static int whereLoopAddBtree( ** index scans so that a covering index scan will be favored over ** a table scan. */ pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; + whereLoopOutputAdjust(pWC, pNew, pSrc->iCursor); rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; if( rc ) break; }else{ Bitmask m = pSrc->colUsed & ~columnsInIndex(pProbe); @@ -4659,7 +4697,9 @@ static int whereLoopAddBtree( ** which we will simplify to just N*log2(N) */ pNew->rRun = rSize + rLogSize; } + whereLoopOutputAdjust(pWC, pNew, pSrc->iCursor); rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; if( rc ) break; } } diff --git a/test/analyze9.test b/test/analyze9.test index 604f340bf5..42bf3d5088 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -805,6 +805,7 @@ do_test 16.1 { ANALYZE; } set nByte2 [lindex [sqlite3_db_status db SCHEMA_USED 0] 1] +puts "$nByte $nByte2" expr {$nByte2 > $nByte+950 && $nByte2 < $nByte+1050} } {1} @@ -950,4 +951,3 @@ for {set i 0} {$i<16} {incr i} { } finish_test -