Consider doing a partial table scan to fulfill an IN operator rather

than using an index.  Try to pick the plan with the lowest cost.

FossilOrigin-Name: 1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b

manifest

@@ -1,5 +1,5 @@
-C The\sIN-early-out\soptimization:\s\sWhen\sdoing\sa\slook-up\son\sa\smulti-column\sindex\nand\san\sIN\soperator\sis\sused\son\sa\scolumn\sother\sthan\sthe\sleft-most\scolumn,\sthen\nif\sno\srows\smatch\sagainst\sthe\sfirst\sIN\svalue,\scheck\sto\smake\ssure\sthere\sexist\nrows\sthat\smatch\sthe\scolumns\sto\sthe\sright\sbefore\scontinuing\swith\sthe\snext\sIN\nvalue.
-D 2018-06-07T18:13:49.091
+C Consider\sdoing\sa\spartial\stable\sscan\sto\sfulfill\san\sIN\soperator\srather\nthan\susing\san\sindex.\s\sTry\sto\spick\sthe\splan\swith\sthe\slowest\scost.
+D 2018-06-08T18:22:10.089
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da
@@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 7dcb13bbcfd8c926546946556014c8f5aa0829eb8b65a6c18f8d187d265200a5
+F src/where.c 8e95858b398f7451c30ed05dd372f39767f04f423fa1d15a19d568268ae9a557
 F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6
 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96
 F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a
@@ -1731,8 +1731,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bb87c054b1b76959e46258ac66b24027f468b390a4148ac67f208a1fbeda4060 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55
-R fe7d026f8be3134e2723fc39eeec5ed5
-T +closed 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55
+P 09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21
+R e63ef1b65ea2f09e13127dc0f6a98d71
+T *branch * in-scan-vs-index
+T *sym-in-scan-vs-index *
+T -sym-trunk *
 U drh
-Z 8a623af2a2288d1935dd9b3507daa5d8
+Z f877a3ca3a67b98847cd7a3f31dd6a75

manifest.uuid

@@ -1 +1 @@
-09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21
+1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b

src/where.c

@@ -2451,7 +2451,7 @@ static int whereLoopAddBtreeIndex(
 
     if( eOp & WO_IN ){
       Expr *pExpr = pTerm->pExpr;
-      pNew->wsFlags |= WHERE_COLUMN_IN;
+      LogEst M, logK;
       if( ExprHasProperty(pExpr, EP_xIsSelect) ){
         /* "x IN (SELECT ...)":  TUNING: the SELECT returns 25 rows */
         int i;
@@ -2471,6 +2471,36 @@ static int whereLoopAddBtreeIndex(
         assert( nIn>0 );  /* RHS always has 2 or more terms...  The parser
                           ** changes "x IN (?)" into "x=?". */
       }
+      /* Let:
+      **   N = the total number of rows in the table
+      **   K = the number of entries on the right-hand side of the IN operator
+      **   M = the number of rows in the table that match terms to the 
+      **       to the left in the same index.  If the IN operator is on
+      **       the left-most index column, M==N.
+      **
+      ** Given the definitions above, it is better to omit the IN operator
+      ** from the index lookup and instead do a scan of the M elements,
+      ** testing each scanned row against the IN operator separately, if:
+      **
+      **        M*log(K) < K*log(N)
+      **
+      ** Our estimates for M, K, and N might be inaccurate, so we build in
+      ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
+      ** with the index, as using an index has better worst-case behavior.
+      */
+      M = pProbe->aiRowLogEst[saved_nEq+1];
+      logK = sqlite3LogEst(nIn);
+      if( M + logK + 10 < nIn + rLogSize ){
+        WHERETRACE(0x40,
+          ("IN operator costs more than scan on column %d of \"%s\" (%d<%d)\n",
+           saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+        continue;
+      }else{
+        WHERETRACE(0x40,
+          ("IN operator cheaper than scan on column %d of \"%s\" (%d>=%d)\n",
+           saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+      }
+      pNew->wsFlags |= WHERE_COLUMN_IN;
     }else if( eOp & (WO_EQ|WO_IS) ){
       int iCol = pProbe->aiColumn[saved_nEq];
       pNew->wsFlags |= WHERE_COLUMN_EQ;