Initial implementation of the unlikely() SQL function used as a hint to

the query planner.

FossilOrigin-Name: 036fc37a034093a4c6fc190633bd41c2b7230d77

manifest

@@ -1,5 +1,5 @@
-C Combine\sthe\sFuncDef.iPrefEnc\sand\sFuncDef.flags\sfields\sinto\sa\ssingle\nnew\sFuncDef.funcFlags\sfield.
+C Initial\simplementation\sof\sthe\sunlikely()\sSQL\sfunction\sused\sas\sa\shint\sto\nthe\squery\splanner.
-D 2013-09-06T13:10:12.385
+D 2013-09-06T15:23:29.191
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -172,10 +172,10 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c
 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4
 F src/delete.c 2dc64ca360b7d7da481183ea920a813a0c203c97
-F src/expr.c 33ae44812d25799000dbcfd54bb2142b30cdd049
+F src/expr.c d0ed048b3b3c97fddbcccb8df43cc39b89e682c2
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c be866cd8c4fa6cae98ba33109578fd1a3311ee5b
-F src/func.c 7650d35651bb0ca903d24f4e5e944bfdaac9f152
+F src/func.c a297b1f3cf48dd483f5453b7155bc5e8f4ca1fc2
 F src/global.c 5caf4deab621abb45b4c607aad1bd21c20aac759
 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4
 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22
@@ -214,14 +214,14 @@ F src/pragma.c 3aa3d8c8623b7b71c5b1bfb72dcc31fb0c25665f
 F src/prepare.c fa6988589f39af8504a61731614cd4f6ae71554f
 F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b
 F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68
-F src/resolve.c 9d53899cc6e1f4ec0b4632d07e97d57827bf63b9
+F src/resolve.c 748618c3df7f37f2cd33c8834770b1e34f9b1185
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
 F src/select.c 9239586282bd146ec5843a2cde7d54cd7816cf78
 F src/shell.c d920a891ca09b8bd262cced7fb0ab9d723f7a747
 F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
-F src/sqliteInt.h 32e959dba9899360ce4ae3cd6cc77b4a8df277e8
+F src/sqliteInt.h fd9005fab1cd063f5c4e76a2120efc080b43750a
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -290,7 +290,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d
 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73
-F src/where.c 06c249a5137575ecf4d527e5a1dadb1087e4375c
+F src/where.c c9f8ae51797bebfbfa49b9e5bede6fc142b44262
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4
 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab
 F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f
 F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88
-F test/analyze9.test 4a48461d71a7b4a958570c580495a97e842e2700
+F test/analyze9.test e688c7289d3d33b7185e2c2afcac6a9f095563a3
 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944
 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d
 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b
@@ -1111,7 +1111,10 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P a99a53b81e29c9514b85318bea028d1667e5f760
+P 97b10e66e98e84755aa577f8da017bf1aea2056c
-R 8c10a6011eb52c6098e833ca515e6c77
+R fcd7cd29a90a12a870cace66939b8877
+T *branch * unlikely-func
+T *sym-unlikely-func *
+T -sym-trunk *
 U drh
-Z c7e49c79214a66374f550dcdb7a70012
+Z 2a30cee668099b5c8e0e2b447ac67feb

manifest.uuid

@@ -1 +1 @@
-97b10e66e98e84755aa577f8da017bf1aea2056c
+036fc37a034093a4c6fc190633bd41c2b7230d77

src/expr.c

@@ -89,8 +89,16 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){
 ** an expression.
 */
 Expr *sqlite3ExprSkipCollate(Expr *pExpr){
-  while( pExpr && (pExpr->op==TK_COLLATE || pExpr->op==TK_AS) ){
+  while( pExpr ){
-    pExpr = pExpr->pLeft;
+    if( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ){
+      pExpr = pExpr->pLeft;
+    }else if( ExprHasAnyProperty(pExpr, EP_Hint) ){
+      assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
+      assert( pExpr->x.pList->nExpr>0 );
+      pExpr = pExpr->x.pList->a[0].pExpr;
+    }else{
+      break;
+    }
   }
   return pExpr;
 }
@@ -2649,6 +2657,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
         break;
       }
 
+      /* The UNLIKELY() function is a no-op.  The result is the value
+      ** of the first argument.
+      */
+      if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){
+        assert( nFarg>=1 );
+        sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
+        break;
+      }
 
       if( pFarg ){
         r1 = sqlite3GetTempRange(pParse, nFarg);

src/func.c

@@ -418,14 +418,14 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){
 }
 
 /*
-** The COALESCE() and IFNULL() functions are implemented as VDBE code so
+** Some functions like COALESCE() and IFNULL() and UNLIKELY() are implemented
-** that unused argument values do not have to be computed.  However, we
+** as VDBE code so that unused argument values do not have to be computed.
-** still need some kind of function implementation for this routines in
+** However, we still need some kind of function implementation for this
-** the function table.  That function implementation will never be called
+** routines in the function table.  The noopFunc macro provides this.
-** so it doesn't matter what the implementation is.  We might as well use
+** noopFunc will never be called so it doesn't matter what the implementation
-** the "version()" function as a substitute.
+** is.  We might as well use the "version()" function as a substitute.
 */
-#define ifnullFunc versionFunc   /* Substitute function - never called */
+#define noopFunc versionFunc   /* Substitute function - never called */
 
 /*
 ** Implementation of random().  Return a random integer.  
@@ -1659,9 +1659,11 @@ void sqlite3RegisterGlobalFunctions(void){
     FUNCTION(lower,              1, 0, 0, lowerFunc        ),
     FUNCTION(coalesce,           1, 0, 0, 0                ),
     FUNCTION(coalesce,           0, 0, 0, 0                ),
-    FUNCTION2(coalesce,         -1, 0, 0, ifnullFunc,  SQLITE_FUNC_COALESCE),
+    FUNCTION2(coalesce,         -1, 0, 0, noopFunc,  SQLITE_FUNC_COALESCE),
     FUNCTION(hex,                1, 0, 0, hexFunc          ),
-    FUNCTION2(ifnull,            2, 0, 0, ifnullFunc,  SQLITE_FUNC_COALESCE),
+    FUNCTION2(ifnull,            2, 0, 0, noopFunc,  SQLITE_FUNC_COALESCE),
+    FUNCTION2(unlikely,          1, 0, 0, noopFunc,  SQLITE_FUNC_UNLIKELY),
+    FUNCTION2(unlikely,          2, 0, 0, noopFunc,  SQLITE_FUNC_UNLIKELY),
     FUNCTION(random,             0, 0, 0, randomFunc       ),
     FUNCTION(randomblob,         1, 0, 0, randomBlob       ),
     FUNCTION(nullif,             2, 0, 1, nullifFunc       ),

src/resolve.c

@@ -570,6 +570,18 @@ static void notValidCheckConstraint(
 # define notValidCheckConstraint(P,N,M)
 #endif
 
+/*
+** Expression p should encode a floating point value between 1.0 and 0.0.
+** Return 1024 times this value.  Or return -1 if p is not a floating point
+** value between 1.0 and 0.0.
+*/
+static int exprProbability(Expr *p){
+  double r = -1.0;
+  if( p->op!=TK_FLOAT ) return -1;
+  sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8);
+  if( r<0.0 || r>1.0 ) return -1;
+  return (int)(r*1000.0);
+}
 
 /*
 ** This routine is callback for sqlite3WalkExpr().
@@ -683,6 +695,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
         }
       }else{
         is_agg = pDef->xFunc==0;
+        if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){
+          ExprSetProperty(pExpr, EP_Hint);
+          if( n==2 ){
+            pExpr->iTable = exprProbability(pList->a[1].pExpr);
+            if( pExpr->iTable<0 ){
+              sqlite3ErrorMsg(pParse, "second parameter to unlikely() must be "
+                                      "between 0.0 and 1.0");
+              pNC->nErr++;
+            }
+          }else{
+            pExpr->iTable = 100;
+          }             
+        }
       }
 #ifndef SQLITE_OMIT_AUTHORIZATION
       if( pDef ){

src/sqliteInt.h

@@ -1752,7 +1752,8 @@ struct Expr {
 #endif
   int iTable;            /* TK_COLUMN: cursor number of table holding column
                          ** TK_REGISTER: register number
-                         ** TK_TRIGGER: 1 -> new, 0 -> old */
+                         ** TK_TRIGGER: 1 -> new, 0 -> old
+                         ** EP_Hint:  1000 times likelihood */
   ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
                          ** TK_VARIABLE: variable number (always >= 1). */
   i16 iAgg;              /* Which entry in pAggInfo->aCol[] or ->aFunc[] */
@@ -1780,7 +1781,7 @@ struct Expr {
 #define EP_FixedDest  0x0200  /* Result needed in a specific register */
 #define EP_IntValue   0x0400  /* Integer value contained in u.iValue */
 #define EP_xIsSelect  0x0800  /* x.pSelect is valid (otherwise x.pList is) */
-#define EP_Hint       0x1000  /* Not used */
+#define EP_Hint       0x1000  /* The UNLIKELY() SQL function */
 #define EP_Reduced    0x2000  /* Expr struct is EXPR_REDUCEDSIZE bytes only */
 #define EP_TokenOnly  0x4000  /* Expr struct is EXPR_TOKENONLYSIZE bytes only */
 #define EP_Static     0x8000  /* Held in memory not obtained from malloc() */

src/where.c

@@ -52,12 +52,13 @@ typedef struct WhereOrSet WhereOrSet;
 ** Cost X is tracked as 10*log2(X) stored in a 16-bit integer.  The
 ** maximum cost for ordinary tables is 64*(2**63) which becomes 6900.
 ** (Virtual tables can return a larger cost, but let's assume they do not.)
-** So all costs can be stored in a 16-bit unsigned integer without risk
+** So all costs can be stored in a 16-bit integer without risk
 ** of overflow.
 **
 ** Costs are estimates, so no effort is made to compute 10*log2(X) exactly.
-** Instead, a close estimate is used.  Any value of X<=1 is stored as 0.
+** Instead, a close estimate is used.  Any value of X=1 is stored as 0.
-** X=2 is 10.  X=3 is 16.  X=1000 is 99. etc.
+** X=2 is 10.  X=3 is 16.  X=1000 is 99. etc.  Negative values are allowed.
+** A WhereCost of -10 means 0.5.  WhereCost of -20 means 0.25.  And so forth.
 **
 ** The tool/wherecosttest.c source file implements a command-line program
 ** that will convert WhereCosts to integers, convert integers to WhereCosts
@@ -65,7 +66,7 @@ typedef struct WhereOrSet WhereOrSet;
 ** command-line program is a useful utility to have around when working with
 ** this module.
 */
-typedef unsigned short int WhereCost;
+typedef short int WhereCost;
 
 /*
 ** This object contains information needed to implement a single nested
@@ -268,6 +269,7 @@ struct WhereTerm {
     WhereOrInfo *pOrInfo;   /* Extra information if (eOperator & WO_OR)!=0 */
     WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */
   } u;
+  WhereCost truthProb;    /* Probability of truth for this expression */
   u16 eOperator;          /* A WO_xx value describing <op> */
   u8 wtFlags;             /* TERM_xxx bit flags.  See below */
   u8 nChild;              /* Number of children that must disable us */
@@ -642,6 +644,9 @@ static void whereClauseClear(WhereClause *pWC){
   }
 }
 
+/* Forward declaration */
+static WhereCost whereCost(tRowcnt x);
+
 /*
 ** Add a single new WhereTerm entry to the WhereClause object pWC.
 ** The new WhereTerm object is constructed from Expr p and with wtFlags.
@@ -683,6 +688,11 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){
     pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]);
   }
   pTerm = &pWC->a[idx = pWC->nTerm++];
+  if( p && ExprHasAnyProperty(p, EP_Hint) ){
+    pTerm->truthProb = whereCost(p->iTable) - 99;
+  }else{
+    pTerm->truthProb = -1;
+  }
   pTerm->pExpr = sqlite3ExprSkipCollate(p);
   pTerm->wtFlags = wtFlags;
   pTerm->pWC = pWC;
@@ -4257,6 +4267,32 @@ whereLoopInsert_noop:
   return SQLITE_OK;  
 }
 
+/*
+** Adjust the WhereLoop.nOut value downward to account for terms of the
+** WHERE clause that reference the loop but which are not used by an
+** index.
+**
+** In the current implementation, the first extra WHERE clause term reduces
+** the number of output rows by a factor of 10 and each additional term
+** reduces the number of output rows by sqrt(2).
+*/
+static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop, int iCur){
+  WhereTerm *pTerm;
+  Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
+  int x = 0;
+  int i;
+  for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){
+    if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
+    if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
+    if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
+    x += pTerm->truthProb;
+  }
+  for(i=pLoop->nLTerm-1; i>=0; i--){
+    x -= pLoop->aLTerm[i]->truthProb;
+  }
+  if( x<0 ) pLoop->nOut += x;
+}
+
 /*
 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex.
 ** Try to match one more.
@@ -4423,7 +4459,7 @@ static int whereLoopAddBtreeIndex(
     }
     /* Step cost for each output row */
     pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut);
-    /* TBD: Adjust nOut for additional constraints */
+    whereLoopOutputAdjust(pBuilder->pWC, pNew, pSrc->iCursor);
     rc = whereLoopInsert(pBuilder, pNew);
     if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
      && pNew->u.btree.nEq<(pProbe->nColumn + (pProbe->zName!=0))
@@ -4627,7 +4663,9 @@ static int whereLoopAddBtree(
       **     index scans so that a covering index scan will be favored over
       **     a table scan. */
       pNew->rRun = whereCostAdd(rSize,rLogSize) + 16;
+      whereLoopOutputAdjust(pWC, pNew, pSrc->iCursor);
       rc = whereLoopInsert(pBuilder, pNew);
+      pNew->nOut = rSize;
       if( rc ) break;
     }else{
       Bitmask m = pSrc->colUsed & ~columnsInIndex(pProbe);
@@ -4659,7 +4697,9 @@ static int whereLoopAddBtree(
           ** which we will simplify to just N*log2(N) */
           pNew->rRun = rSize + rLogSize;
         }
+        whereLoopOutputAdjust(pWC, pNew, pSrc->iCursor);
         rc = whereLoopInsert(pBuilder, pNew);
+        pNew->nOut = rSize;
         if( rc ) break;
       }
     }

test/analyze9.test

@@ -805,6 +805,7 @@ do_test 16.1 {
     ANALYZE;
   }
   set nByte2 [lindex [sqlite3_db_status db SCHEMA_USED 0] 1]
+puts "$nByte $nByte2"
 
   expr {$nByte2 > $nByte+950 && $nByte2 < $nByte+1050}
 } {1}
@@ -950,4 +951,3 @@ for {set i 0} {$i<16} {incr i} {
 }
 
 finish_test
-