diff --git a/manifest b/manifest
index 1e8f395183..765acd7045 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\s".mode\soff"\sand\s".mode\scount"\sto\sthe\sCLI.
-D 2021-12-09T16:26:45.642
+C Add\ssupport\sfor\sBloom-filters\sas\sa\sperformance\soptimization\sfor\sjoins.
+D 2021-12-09T20:06:18.928
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -515,7 +515,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c
-F src/main.c 1ea70751e6005ab6a9f784730fa0919efaa6639440a287deb73cb711e5aae57a
+F src/main.c 674a0fdfc2808e1d5a78b2eefe2ec3f93428cf82f0f6c013d577df1a1caa5940
F src/malloc.c ef796bcc0e81d845d59a469f1cf235056caf9024172fd524e32136e65593647b
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -552,11 +552,11 @@ F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c
F src/resolve.c 4a1db4aadd802683db40ca2dbbb268187bd195f10cbdb7206dbd8ac988795571
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c a7a3d9f54eb24821ec5f67f2e5589b68a5d42d46fc5849d7376886777d93a85a
-F src/shell.c.in 24e39ae0d6c2b8681740bc88e68ed3247a110c3a7aaaca63416477f3de79bcad
-F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
+F src/shell.c.in 239bee1085d94964f02582b0714dc3fc85cfc16e27e95813e4dbc24bb215a7e0
+F src/sqlite.h.in 5999d6db0e65afbd686b76cddc385b310aa3815624edba43987913067f50e209
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h 375bb03b5c2de4c3e2f746e6205f01b8a51abd5835a1a72ab9a1c027f5c9849e
+F src/sqliteInt.h b4391c3c2ae0a8020ce0f543fc2b529f9bcdf72ab7ba3c31d170e3228169162f
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -621,15 +621,15 @@ F src/trigger.c 2ef56f0b7b75349a5557d0604b475126329c2e1a02432e7d49c4c710613e8254
F src/update.c d6f5c7b9e072660757ac7d58175aca11c07cb95ebbb297ae7f38853700f52328
F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167
+F src/util.c 569349b0bddcbfbc661856f446adb92e1b0a47b3cbef548da9fc5aa639d7964c
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c f25438ea079ff2a363c2cc0b70f3178fcf48d3c82ebe72534736a702a4d0eb54
+F src/vdbe.c 5c05b1149a930851f36b29172002dfa56ce703e13f82d5bb4242a29dc144e350
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
-F src/vdbeInt.h 31fbabdc1ed61d9695337dfe5269ea94e1cf615c17f5cafeaa1bb01066820bab
+F src/vdbeInt.h 910985ac2783fe0938b314d811759d53fd25caf215810f62ca1ff068d6d60d7b
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
F src/vdbeaux.c 21db442d159fd745a7693d157b5f998260b6af4ca60de559fa3b7b68c7405af2
F src/vdbeblob.c 29c4118f7ee615cdee829e8401f6ead1b96b95d545b4de0042f6de39c962c652
-F src/vdbemem.c a3d91dc9bb9ef725db77e4e9de7e1acef43192c9f8406c307665d503e3c2837c
+F src/vdbemem.c da4d594084d581be6436582bb44bb128feeb138a3e6c313eda6749ebdc3a65ec
F src/vdbesort.c 513b481c8bab4a6578c92194a60cf3bc3b48736e4a53f8d2d7918121c5b594e7
F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c
@@ -638,9 +638,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 6e07a2ebfccedec1926f9cd13f773741002a56c40dd90adc3ea25e41354db46f
-F src/whereInt.h 14ebb040acac47091a4dd3075f4ab511cad7fc31010fc5e6a750e06b7950c021
-F src/wherecode.c 280f1f87311827f8921a815dbb9b8e84bb394556716fa6a2e111c855b299de20
+F src/where.c 71bbbac8748ddd470dd3c4213342dce2e39c2c0aa84f9b03d80bbc075fb7c502
+F src/whereInt.h e83f7ba73db5b1b2685118fad67d178fbe04751a25419f0f6ff73e58b4807325
+F src/wherecode.c 98ec56212ea7c0993f5eb808c515076d42c814f833c1641022927903cef0e811
F src/whereexpr.c 791544603b254cf11f8e84e3b50b0863c57322e9f213b828680f658e232ebc57
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1138,7 +1138,7 @@ F test/join.test 25da4f53523a4aa17c893134b47fba6aa4799bb33350517b157785878290e23
F test/join2.test 9bdc615841b91c97a16d68bad9508aea11fa0c6b34e5689847bcc4dac70e4990
F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
-F test/join5.test f418fccdfefa41f1659663463aa517431ddcf3e30ccbb80e64173b7d615a03f4
+F test/join5.test d22395f7d4020a58cabbc8316f300a5cfef84aee9e8ba7ce79b33cc43a3e1e2e
F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c
F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
@@ -1856,7 +1856,7 @@ F tool/lemon.c 258881835bd5bccd0c74fb110fe54244ff18e8e7ef3d949cbdab7187f02132bb
F tool/lempar.c 57478ea48420da05faa873c6d1616321caa5464644588c97fbe8e0ea04450748
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
-F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
+F tool/logest.c 83dbfda91615f1db5dce38215303d8bb456f437342d2c64262406dbdd1c931e2
F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176
F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8
@@ -1934,8 +1934,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7046877006259a0073cca4ddb4541321d373c462eac9924be26b5ae224075732
-Q +b11f4d080aa9e6f694e2ec401e871f42bf25997e8e8bf77fa9b6014a50466e3c
-R c56a6044af5ef1581ccf8dda1f9e99a9
+P 1eefd957ff35e961685db565f7ef116c566a04574c5bedad7298b3cc69dd72d2 ce42039f5647b1f276acf5d9911528ecb47df1544a587def72c8cd6b2f664289
+R c6e1a17bdb20d0994b97369ef3d14311
+T +closed ce42039f5647b1f276acf5d9911528ecb47df1544a587def72c8cd6b2f664289
U drh
-Z 806d5f1c1ccd0354c71d5e148acabb04
+Z 2960f76524291316688b632931fc780c
diff --git a/manifest.uuid b/manifest.uuid
index a61c9e96ac..bbb228f8b8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1eefd957ff35e961685db565f7ef116c566a04574c5bedad7298b3cc69dd72d2
\ No newline at end of file
+633bfeeea2bccdd44126acf3f61ecca163c9d933bdc787a2c18a697dc9406882
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 804719f176..1b7853f609 100644
--- a/src/main.c
+++ b/src/main.c
@@ -4347,6 +4347,26 @@ int sqlite3_test_control(int op, ...){
break;
}
+ /* sqlite3_test_control(SQLITE_TESTCTRL_LOGEST,
+ ** double fIn, // Input value
+ ** int *pLogEst, // sqlite3LogEstFromDouble(fIn)
+ ** u64 *pInt, // sqlite3LogEstToInt(*pLogEst)
+ ** int *pLogEst2 // sqlite3LogEst(*pInt)
+ ** );
+ **
+ ** Test access for the LogEst conversion routines.
+ */
+ case SQLITE_TESTCTRL_LOGEST: {
+ double rIn = va_arg(ap, double);
+ LogEst rLogEst = sqlite3LogEstFromDouble(rIn);
+ u64 iInt = sqlite3LogEstToInt(rLogEst);
+ va_arg(ap, int*)[0] = rLogEst;
+ va_arg(ap, u64*)[0] = iInt;
+ va_arg(ap, int*)[0] = sqlite3LogEst(iInt);
+ break;
+ }
+
+
#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD)
/* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue)
**
diff --git a/src/shell.c.in b/src/shell.c.in
index 216e0e46ff..3262f98c12 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2798,6 +2798,7 @@ static int display_stats(
}
if( pArg->pStmt ){
+ int iHit, iMiss;
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP,
bReset);
raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur);
@@ -2805,6 +2806,12 @@ static int display_stats(
raw_printf(pArg->out, "Sort Operations: %d\n", iCur);
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset);
raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur);
+ iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset);
+ iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset);
+ if( iHit || iMiss ){
+ raw_printf(pArg->out, "Bloom filter bypass taken: %d/%d\n",
+ iHit, iHit+iMiss);
+ }
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset);
raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur);
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset);
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 98a028b0b0..63a6f37e9a 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -7944,7 +7944,8 @@ int sqlite3_test_control(int op, ...);
#define SQLITE_TESTCTRL_SEEK_COUNT 30
#define SQLITE_TESTCTRL_TRACEFLAGS 31
#define SQLITE_TESTCTRL_TUNE 32
-#define SQLITE_TESTCTRL_LAST 32 /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_LOGEST 33
+#define SQLITE_TESTCTRL_LAST 33 /* Largest TESTCTRL */
/*
** CAPI3REF: SQL Keyword Checking
@@ -8467,6 +8468,16 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
** The counter is incremented on the first [sqlite3_step()] call of each
** cycle.
**
+** [[SQLITE_STMTSTATUS_FILTER_MISS]]
+** [[SQLITE_STMTSTATUS_FILTER HIT]]
+** <dt>SQLITE_STMTSTATUS_FILTER_HIT<br>
+** SQLITE_STMTSTATUS_FILTER_MISS</dt>
+** <dd>^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join
+** step was bypassed because a Bloom filter returned not-found. The
+** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of
+** times that the Bloom filter returned a find, and thus the join step
+** had to be processed as normal.
+**
** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt>
** <dd>^This is the approximate number of bytes of heap memory
** used to store the prepared statement. ^This value is not actually
@@ -8481,6 +8492,8 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
#define SQLITE_STMTSTATUS_VM_STEP 4
#define SQLITE_STMTSTATUS_REPREPARE 5
#define SQLITE_STMTSTATUS_RUN 6
+#define SQLITE_STMTSTATUS_FILTER_MISS 7
+#define SQLITE_STMTSTATUS_FILTER_HIT 8
#define SQLITE_STMTSTATUS_MEMUSED 99
/*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index c1a0d9853e..bd02e2626f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1761,6 +1761,8 @@ struct sqlite3 {
#define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */
#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */
/* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */
+#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */
+#define SQLITE_BloomPulldown 0x00100000 /* Run Bloom filters early */
#define SQLITE_AllOpts 0xffffffff /* All optimizations */
/*
@@ -4774,14 +4776,8 @@ int sqlite3Utf8CharLen(const char *pData, int nByte);
u32 sqlite3Utf8Read(const u8**);
LogEst sqlite3LogEst(u64);
LogEst sqlite3LogEstAdd(LogEst,LogEst);
-#ifndef SQLITE_OMIT_VIRTUALTABLE
LogEst sqlite3LogEstFromDouble(double);
-#endif
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
- defined(SQLITE_ENABLE_STAT4) || \
- defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
u64 sqlite3LogEstToInt(LogEst);
-#endif
VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int);
const char *sqlite3VListNumToName(VList*,int);
int sqlite3VListNameToNum(VList*,const char*,int);
diff --git a/src/util.c b/src/util.c
index 8452aea665..8ea951fa16 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1586,7 +1586,6 @@ LogEst sqlite3LogEst(u64 x){
return a[x&7] + y - 10;
}
-#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Convert a double into a LogEst
** In other words, compute an approximation for 10*log2(x).
@@ -1601,16 +1600,9 @@ LogEst sqlite3LogEstFromDouble(double x){
e = (a>>52) - 1022;
return e*10;
}
-#endif /* SQLITE_OMIT_VIRTUALTABLE */
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
- defined(SQLITE_ENABLE_STAT4) || \
- defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
/*
** Convert a LogEst into an integer.
-**
-** Note that this routine is only used when one or more of various
-** non-standard compile-time options is enabled.
*/
u64 sqlite3LogEstToInt(LogEst x){
u64 n;
@@ -1618,17 +1610,9 @@ u64 sqlite3LogEstToInt(LogEst x){
x /= 10;
if( n>=5 ) n -= 2;
else if( n>=1 ) n -= 1;
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
- defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
if( x>60 ) return (u64)LARGEST_INT64;
-#else
- /* If only SQLITE_ENABLE_STAT4 is on, then the largest input
- ** possible to this routine is 310, resulting in a maximum x of 31 */
- assert( x<=60 );
-#endif
return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x);
}
-#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */
/*
** Add a new name/number pair to a VList. This might require that the
diff --git a/src/vdbe.c b/src/vdbe.c
index 7dfe379175..27e030ec19 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -671,6 +671,31 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){
}
}
+/*
+** Compute a bloom filter hash using pOp->p4.i registers from aMem[] beginning
+** with pOp->p3. Return the hash.
+*/
+static u64 filterHash(const Mem *aMem, const Op *pOp){
+ int i, mx;
+ u64 h = 0;
+
+ i = pOp->p3;
+ assert( pOp->p4type==P4_INT32 );
+ mx = i + pOp->p4.i;
+ for(i=pOp->p3, mx=i+pOp->p4.i; i<mx; i++){
+ const Mem *p = &aMem[i];
+ if( p->flags & (MEM_Int|MEM_IntReal) ){
+ h += p->u.i;
+ }else if( p->flags & MEM_Real ){
+ h += sqlite3VdbeIntValue(p);
+ }else if( p->flags & (MEM_Str|MEM_Blob) ){
+ h += p->n;
+ if( p->flags & MEM_Zero ) h += p->u.nZero;
+ }
+ }
+ return h;
+}
+
/*
** Return the symbolic name for the data type of a pMem
*/
@@ -1325,12 +1350,18 @@ case OP_SoftNull: {
** Synopsis: r[P2]=P4 (len=P1)
**
** P4 points to a blob of data P1 bytes long. Store this
-** blob in register P2.
+** blob in register P2. If P4 is a NULL pointer, then construct
+** a zero-filled blob that is P1 bytes long in P2.
*/
case OP_Blob: { /* out2 */
assert( pOp->p1 <= SQLITE_MAX_LENGTH );
pOut = out2Prerelease(p, pOp);
- sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0);
+ if( pOp->p4.z==0 ){
+ sqlite3VdbeMemSetZeroBlob(pOut, pOp->p1);
+ if( sqlite3VdbeMemExpandBlob(pOut) ) goto no_mem;
+ }else{
+ sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0);
+ }
pOut->enc = encoding;
UPDATE_MAX_BLOBSIZE(pOut);
break;
@@ -3341,7 +3372,7 @@ case OP_MakeRecord: {
break;
}
-/* Opcode: Count P1 P2 p3 * *
+/* Opcode: Count P1 P2 P3 * *
** Synopsis: r[P2]=count()
**
** Store the number of entries (an integer value) in the table or index
@@ -8127,6 +8158,77 @@ case OP_Function: { /* group */
break;
}
+/* Opcode: FilterAdd P1 * P3 P4 *
+** Synopsis: filter(P1) += key(P3@P4)
+**
+** Compute a hash on the P4 registers starting with r[P3] and
+** add that hash to the bloom filter contained in r[P1].
+*/
+case OP_FilterAdd: {
+ u64 h;
+
+ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
+ pIn1 = &aMem[pOp->p1];
+ assert( pIn1->flags & MEM_Blob );
+ assert( pIn1->n>0 );
+ h = filterHash(aMem, pOp);
+#ifdef SQLITE_DEBUG
+ if( db->flags&SQLITE_VdbeTrace ){
+ int ii;
+ for(ii=pOp->p3; ii<pOp->p3+pOp->p4.i; ii++){
+ registerTrace(ii, &aMem[ii]);
+ }
+ printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n));
+ }
+#endif
+ h %= pIn1->n;
+ pIn1->z[h/8] |= 1<<(h&7);
+ break;
+}
+
+/* Opcode: Filter P1 P2 P3 P4 *
+** Synopsis: if key(P3@P4) not in filter(P1) goto P2
+**
+** Compute a hash on the key contained in the P4 registers starting
+** with r[P3]. Check to see if that hash is found in the
+** bloom filter hosted by register P1. If it is not present then
+** maybe jump to P2. Otherwise fall through.
+**
+** False negatives are harmless. It is always safe to fall through,
+** even if the value is in the bloom filter. A false negative causes
+** more CPU cycles to be used, but it should still yield the correct
+** answer. However, an incorrect answer may well arise from a
+** false positive - if the jump is taken when it should fall through.
+*/
+case OP_Filter: { /* jump */
+ u64 h;
+
+ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
+ pIn1 = &aMem[pOp->p1];
+ assert( (pIn1->flags & MEM_Blob)!=0 );
+ assert( pIn1->n >= 1 );
+ h = filterHash(aMem, pOp);
+#ifdef SQLITE_DEBUG
+ if( db->flags&SQLITE_VdbeTrace ){
+ int ii;
+ for(ii=pOp->p3; ii<pOp->p3+pOp->p4.i; ii++){
+ registerTrace(ii, &aMem[ii]);
+ }
+ printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n));
+ }
+#endif
+ h %= pIn1->n;
+ if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){
+ VdbeBranchTaken(1, 2);
+ p->aCounter[SQLITE_STMTSTATUS_FILTER_HIT]++;
+ goto jump_to_p2;
+ }else{
+ p->aCounter[SQLITE_STMTSTATUS_FILTER_MISS]++;
+ VdbeBranchTaken(0, 2);
+ }
+ break;
+}
+
/* Opcode: Trace P1 P2 * P4 *
**
** Write P4 on the statement trace output if statement tracing is
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 599d064165..38863f6d65 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -430,7 +430,7 @@ struct Vdbe {
bft bIsReader:1; /* True for statements that read */
yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */
yDbMask lockMask; /* Subset of btreeMask that requires a lock */
- u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */
+ u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */
char *zSql; /* Text of the SQL statement that generated this */
#ifdef SQLITE_ENABLE_NORMALIZE
char *zNormSql; /* Normalization of the associated SQL statement */
@@ -538,7 +538,7 @@ int sqlite3VdbeMemSetRowSet(Mem*);
int sqlite3VdbeMemMakeWriteable(Mem*);
int sqlite3VdbeMemStringify(Mem*, u8, u8);
int sqlite3IntFloatCompare(i64,double);
-i64 sqlite3VdbeIntValue(Mem*);
+i64 sqlite3VdbeIntValue(const Mem*);
int sqlite3VdbeMemIntegerify(Mem*);
double sqlite3VdbeRealValue(Mem*);
int sqlite3VdbeBooleanValue(Mem*, int ifNull);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 570a2eb38c..5a9d15f465 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -596,12 +596,12 @@ static SQLITE_NOINLINE i64 doubleToInt64(double r){
**
** If pMem represents a string value, its encoding might be changed.
*/
-static SQLITE_NOINLINE i64 memIntValue(Mem *pMem){
+static SQLITE_NOINLINE i64 memIntValue(const Mem *pMem){
i64 value = 0;
sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc);
return value;
}
-i64 sqlite3VdbeIntValue(Mem *pMem){
+i64 sqlite3VdbeIntValue(const Mem *pMem){
int flags;
assert( pMem!=0 );
assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
diff --git a/src/where.c b/src/where.c
index 649b786520..be75101b26 100644
--- a/src/where.c
+++ b/src/where.c
@@ -717,9 +717,9 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
** index existed.
*/
static int termCanDriveIndex(
- WhereTerm *pTerm, /* WHERE clause term to check */
- SrcItem *pSrc, /* Table we are trying to access */
- Bitmask notReady /* Tables in outer loops of the join */
+ const WhereTerm *pTerm, /* WHERE clause term to check */
+ const SrcItem *pSrc, /* Table we are trying to access */
+ const Bitmask notReady /* Tables in outer loops of the join */
){
char aff;
if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
@@ -750,11 +750,11 @@ static int termCanDriveIndex(
** and to set up the WhereLevel object pLevel so that the code generator
** makes use of the automatic index.
*/
-static void constructAutomaticIndex(
+static SQLITE_NOINLINE void constructAutomaticIndex(
Parse *pParse, /* The parsing context */
- WhereClause *pWC, /* The WHERE clause */
- SrcItem *pSrc, /* The FROM clause term to get the next index */
- Bitmask notReady, /* Mask of cursors that are not available */
+ const WhereClause *pWC, /* The WHERE clause */
+ const SrcItem *pSrc, /* The FROM clause term to get the next index */
+ const Bitmask notReady, /* Mask of cursors that are not available */
WhereLevel *pLevel /* Write new index here */
){
int nKeyCol; /* Number of columns in the constructed index */
@@ -909,6 +909,10 @@ static void constructAutomaticIndex(
sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
VdbeComment((v, "for %s", pTable->zName));
+ if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){
+ pLevel->regFilter = ++pParse->nMem;
+ sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
+ }
/* Fill the automatic index with content */
pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom];
@@ -931,6 +935,10 @@ static void constructAutomaticIndex(
regBase = sqlite3GenerateIndexKey(
pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
);
+ if( pLevel->regFilter ){
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
+ regBase, pLoop->u.btree.nEq);
+ }
sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
@@ -957,6 +965,123 @@ end_auto_index_create:
}
#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
+/*
+** Generate bytecode that will initialize a Bloom filter that is appropriate
+** for pLevel.
+**
+** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
+** flag set, initialize a Bloomfilter for them as well. Except don't do
+** this recursive initialization if the SQLITE_BloomPulldown optimization has
+** been turned off.
+**
+** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
+** from the loop, but the regFilter value is set to a register that implements
+** the Bloom filter. When regFilter is positive, the
+** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
+** and skip the subsequence B-Tree seek if the Bloom filter indicates that
+** no matching rows exist.
+**
+** This routine may only be called if it has previously been determined that
+** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
+** is set.
+*/
+static SQLITE_NOINLINE void constructBloomFilter(
+ WhereInfo *pWInfo, /* The WHERE clause */
+ int iLevel, /* Index in pWInfo->a[] that is pLevel */
+ WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
+ Bitmask notReady /* Loops that are not ready */
+){
+ int addrOnce; /* Address of opening OP_Once */
+ int addrTop; /* Address of OP_Rewind */
+ int addrCont; /* Jump here to skip a row */
+ const WhereTerm *pTerm; /* For looping over WHERE clause terms */
+ const WhereTerm *pWCEnd; /* Last WHERE clause term */
+ Parse *pParse = pWInfo->pParse; /* Parsing context */
+ Vdbe *v = pParse->pVdbe; /* VDBE under construction */
+ WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
+ int iCur; /* Cursor for table getting the filter */
+
+ assert( pLoop!=0 );
+ assert( v!=0 );
+ assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
+
+ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
+ do{
+ const SrcItem *pItem;
+ const Table *pTab;
+ u64 sz;
+ sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
+ addrCont = sqlite3VdbeMakeLabel(pParse);
+ iCur = pLevel->iTabCur;
+ pLevel->regFilter = ++pParse->nMem;
+
+ /* The Bloom filter is a Blob held in a register. Initialize it
+ ** to zero-filled blob of at least 80K bits, but maybe more if the
+ ** estimated size of the table is larger. We could actually
+ ** measure the size of the table at run-time using OP_Count with
+ ** P3==1 and use that value to initialize the blob. But that makes
+ ** testing complicated. By basing the blob size on the value in the
+ ** sqlite_stat1 table, testing is much easier.
+ */
+ pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+ assert( pItem!=0 );
+ pTab = pItem->pTab;
+ assert( pTab!=0 );
+ sz = sqlite3LogEstToInt(pTab->nRowLogEst);
+ if( sz<10000 ){
+ sz = 10000;
+ }else if( sz>10000000 ){
+ sz = 10000000;
+ }
+ sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
+
+ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
+ pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
+ for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
+ Expr *pExpr = pTerm->pExpr;
+ if( (pTerm->wtFlags & TERM_VIRTUAL)==0
+ && sqlite3ExprIsTableConstant(pExpr, iCur)
+ ){
+ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
+ }
+ }
+ if( pLoop->wsFlags & WHERE_IPK ){
+ int r1 = sqlite3GetTempReg(pParse);
+ sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
+ sqlite3ReleaseTempReg(pParse, r1);
+ }else{
+ Index *pIdx = pLoop->u.btree.pIndex;
+ int n = pLoop->u.btree.nEq;
+ int r1 = sqlite3GetTempRange(pParse, n);
+ int jj;
+ for(jj=0; jj<n; jj++){
+ int iCol = pIdx->aiColumn[jj];
+ assert( pIdx->pTable==pItem->pTab );
+ sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
+ }
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
+ sqlite3ReleaseTempRange(pParse, r1, n);
+ }
+ sqlite3VdbeResolveLabel(v, addrCont);
+ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
+ VdbeCoverage(v);
+ sqlite3VdbeJumpHere(v, addrTop);
+ pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
+ if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
+ while( iLevel < pWInfo->nLevel ){
+ iLevel++;
+ pLevel = &pWInfo->a[iLevel];
+ pLoop = pLevel->pWLoop;
+ if( pLoop==0 ) continue;
+ if( pLoop->prereq & notReady ) continue;
+ if( pLoop->wsFlags & WHERE_BLOOMFILTER ) break;
+ }
+ }while( iLevel < pWInfo->nLevel );
+ sqlite3VdbeJumpHere(v, addrOnce);
+}
+
+
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Allocate and populate an sqlite3_index_info structure. It is the
@@ -1899,9 +2024,9 @@ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
sqlite3_free(z);
}
if( p->wsFlags & WHERE_SKIPSCAN ){
- sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
+ sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
}else{
- sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm);
+ sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
}
sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){
@@ -2373,6 +2498,13 @@ static void whereLoopOutputAdjust(
if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
}
if( j<0 ){
+ if( pLoop->maskSelf==pTerm->prereqAll ){
+ /* If there are extra terms in the WHERE clause not used by an index
+ ** that depend only on the table being scanned, and that will tend to
+ ** cause many rows to be omitted, then mark that table as
+ ** "self-culling". */
+ pLoop->wsFlags |= WHERE_SELFCULL;
+ }
if( pTerm->truthProb<=0 ){
/* If a truth probability is specified using the likelihood() hints,
** then use the probability provided by the application. */
@@ -2400,7 +2532,9 @@ static void whereLoopOutputAdjust(
}
}
}
- if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce;
+ if( pLoop->nOut > nRow-iReduce ){
+ pLoop->nOut = nRow - iReduce;
+ }
}
/*
@@ -4834,6 +4968,60 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
return notReady;
}
+/*
+** Check to see if there are any SEARCH loops that might benefit from
+** using a Bloom filter. Consider a Bloom filter if:
+**
+** (1) The SEARCH happens more than N times where N is the number
+** of rows in the table that is being considered for the Bloom
+** filter.
+** (2) Some searches are expected to find zero rows. (This is determined
+** by the WHERE_SELFCULL flag on the term.)
+** (3) Bloom-filter processing is not disabled. (Checked by the
+** caller.)
+** (4) The size of the table being searched is known by ANALYZE.
+**
+** This block of code merely checks to see if a Bloom filter would be
+** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
+** WhereLoop. The implementation of the Bloom filter comes further
+** down where the code for each WhereLoop is generated.
+*/
+static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
+ const WhereInfo *pWInfo
+){
+ int i;
+ LogEst nSearch;
+
+ assert( pWInfo->nLevel>=2 );
+ assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
+ nSearch = pWInfo->a[0].pWLoop->nOut;
+ for(i=1; i<pWInfo->nLevel; i++){
+ WhereLoop *pLoop = pWInfo->a[i].pWLoop;
+ const int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
+ if( (pLoop->wsFlags & reqFlags)==reqFlags
+ /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
+ && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
+ ){
+ SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
+ Table *pTab = pItem->pTab;
+ pTab->tabFlags |= TF_StatsUsed;
+ if( nSearch > pTab->nRowLogEst
+ && (pTab->tabFlags & TF_HasStat1)!=0
+ ){
+ testcase( pItem->fg.jointype & JT_LEFT );
+ pLoop->wsFlags |= WHERE_BLOOMFILTER;
+ pLoop->wsFlags &= ~WHERE_IDX_ONLY;
+ WHERETRACE(0xffff, (
+ "-> use Bloom-filter on loop %c because there are ~%.1e "
+ "lookups into %s which has only ~%.1e rows\n",
+ pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
+ (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
+ }
+ }
+ nSearch += pLoop->nOut;
+ }
+}
+
/*
** Generate the beginning of the loop used for WHERE clause processing.
** The return value is a pointer to an opaque structure that contains
@@ -5223,6 +5411,15 @@ WhereInfo *sqlite3WhereBegin(
assert( nTabList>0 );
}
+ /* Check to see if there are any SEARCH loops that might benefit from
+ ** using a Bloom filter.
+ */
+ if( pWInfo->nLevel>=2
+ && OptimizationEnabled(db, SQLITE_BloomFilter)
+ ){
+ whereCheckIfBloomFilterIsUseful(pWInfo);
+ }
+
#if defined(WHERETRACE_ENABLED)
if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
@@ -5411,13 +5608,17 @@ WhereInfo *sqlite3WhereBegin(
if( pParse->nErr ) goto whereBeginError;
pLevel = &pWInfo->a[ii];
wsFlags = pLevel->pWLoop->wsFlags;
+ if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
+ if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
- if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){
- constructAutomaticIndex(pParse, &pWInfo->sWC,
- &pTabList->a[pLevel->iFrom], notReady, pLevel);
+ constructAutomaticIndex(pParse, &pWInfo->sWC,
+ &pTabList->a[pLevel->iFrom], notReady, pLevel);
+#endif
+ }else{
+ constructBloomFilter(pWInfo, ii, pLevel, notReady);
+ }
if( db->mallocFailed ) goto whereBeginError;
}
-#endif
addrExplain = sqlite3WhereExplainOneScan(
pParse, pTabList, pLevel, wctrlFlags
);
diff --git a/src/whereInt.h b/src/whereInt.h
index 0df84ad6c0..8051b78a02 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -64,6 +64,7 @@ struct WhereLevel {
u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */
int addrLikeRep; /* LIKE range processing address */
#endif
+ int regFilter; /* Bloom filter */
u8 iFrom; /* Which entry in the FROM clause */
u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */
int p1, p2; /* Operands of the opcode used to end the loop */
@@ -506,8 +507,14 @@ int sqlite3WhereExplainOneScan(
WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */
u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */
);
+int sqlite3WhereExplainBloomFilter(
+ const Parse *pParse, /* Parse context */
+ const WhereInfo *pWInfo, /* WHERE clause */
+ const WhereLevel *pLevel /* Bloom filter on this level */
+);
#else
# define sqlite3WhereExplainOneScan(u,v,w,x) 0
+# define sqlite3WhereExplainBloomFilter(u,v,w) 0
#endif /* SQLITE_OMIT_EXPLAIN */
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
void sqlite3WhereAddScanStatus(
@@ -600,5 +607,7 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
#define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */
#define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */
#define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */
+#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */
+#define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */
#endif /* !defined(SQLITE_WHEREINT_H) */
diff --git a/src/wherecode.c b/src/wherecode.c
index a6633967a2..dd73bc4e83 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -176,19 +176,27 @@ int sqlite3WhereExplainOneScan(
explainIndexRange(&str, pLoop);
}
}else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){
- const char *zRangeOp;
+ char cRangeOp;
+#if 0 /* Better output, but breaks many tests */
+ const Table *pTab = pItem->pTab;
+ const char *zRowid = pTab->iPKey>=0 ? pTab->aCol[pTab->iPKey].zCnName:
+ "rowid";
+#else
+ const char *zRowid = "rowid";
+#endif
+ sqlite3_str_appendf(&str, " USING INTEGER PRIMARY KEY (%s", zRowid);
if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){
- zRangeOp = "=";
+ cRangeOp = '=';
}else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
- zRangeOp = ">? AND rowid<";
+ sqlite3_str_appendf(&str, ">? AND %s", zRowid);
+ cRangeOp = '<';
}else if( flags&WHERE_BTM_LIMIT ){
- zRangeOp = ">";
+ cRangeOp = '>';
}else{
assert( flags&WHERE_TOP_LIMIT);
- zRangeOp = "<";
+ cRangeOp = '<';
}
- sqlite3_str_appendf(&str,
- " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp);
+ sqlite3_str_appendf(&str, "%c?)", cRangeOp);
}
#ifndef SQLITE_OMIT_VIRTUALTABLE
else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
@@ -211,6 +219,56 @@ int sqlite3WhereExplainOneScan(
}
return ret;
}
+
+/*
+** Add a single OP_Explain opcode that describes a Bloom filter.
+**
+** Or if not processing EXPLAIN QUERY PLAN and not in a SQLITE_DEBUG and/or
+** SQLITE_ENABLE_STMT_SCANSTATUS build, then OP_Explain opcodes are not
+** required and this routine is a no-op.
+**
+** If an OP_Explain opcode is added to the VM, its address is returned.
+** Otherwise, if no OP_Explain is coded, zero is returned.
+*/
+int sqlite3WhereExplainBloomFilter(
+ const Parse *pParse, /* Parse context */
+ const WhereInfo *pWInfo, /* WHERE clause */
+ const WhereLevel *pLevel /* Bloom filter on this level */
+){
+ int ret = 0;
+ SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+ Vdbe *v = pParse->pVdbe; /* VM being constructed */
+ sqlite3 *db = pParse->db; /* Database handle */
+ char *zMsg; /* Text to add to EQP output */
+ int i; /* Loop counter */
+ WhereLoop *pLoop; /* The where loop */
+ StrAccum str; /* EQP output string */
+ char zBuf[100]; /* Initial space for EQP output string */
+
+ sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
+ str.printfFlags = SQLITE_PRINTF_INTERNAL;
+ sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem);
+ pLoop = pLevel->pWLoop;
+ if( pLoop->wsFlags & WHERE_IPK ){
+ const Table *pTab = pItem->pTab;
+ if( pTab->iPKey>=0 ){
+ sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName);
+ }else{
+ sqlite3_str_appendf(&str, "rowid=?");
+ }
+ }else{
+ for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
+ const char *z = explainIndexColumnName(pLoop->u.btree.pIndex, i);
+ if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
+ sqlite3_str_appendf(&str, "%s=?", z);
+ }
+ }
+ sqlite3_str_append(&str, ")", 1);
+ zMsg = sqlite3StrAccumFinish(&str);
+ ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
+ pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
+ return ret;
+}
#endif /* SQLITE_OMIT_EXPLAIN */
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -1301,6 +1359,63 @@ static void whereApplyPartialIndexConstraints(
}
}
+/*
+** This routine is called right after An OP_Filter has been generated and
+** before the corresponding index search has been performed. This routine
+** checks to see if there are additional Bloom filters in inner loops that
+** can be checked prior to doing the index lookup. If there are available
+** inner-loop Bloom filters, then evaluate those filters now, before the
+** index lookup. The idea is that a Bloom filter check is way faster than
+** an index lookup, and the Bloom filter might return false, meaning that
+** the index lookup can be skipped.
+**
+** We know that an inner loop uses a Bloom filter because it has the
+** WhereLevel.regFilter set. If an inner-loop Bloom filter is checked,
+** then clear the WhereLevel.regFilter value to prevent the Bloom filter
+** from being checked a second time when the inner loop is evaluated.
+*/
+static SQLITE_NOINLINE void filterPullDown(
+ Parse *pParse, /* Parsing context */
+ WhereInfo *pWInfo, /* Complete information about the WHERE clause */
+ int iLevel, /* Which level of pWInfo->a[] should be coded */
+ int addrNxt, /* Jump here to bypass inner loops */
+ Bitmask notReady /* Loops that are not ready */
+){
+ while( ++iLevel < pWInfo->nLevel ){
+ WhereLevel *pLevel = &pWInfo->a[iLevel];
+ WhereLoop *pLoop = pLevel->pWLoop;
+ if( pLevel->regFilter==0 ) continue;
+ /* ,--- Because constructBloomFilter() has will not have set
+ ** vvvvv--' pLevel->regFilter if this were true. */
+ if( NEVER(pLoop->prereq & notReady) ) continue;
+ if( pLoop->wsFlags & WHERE_IPK ){
+ WhereTerm *pTerm = pLoop->aLTerm[0];
+ int regRowid;
+ assert( pTerm!=0 );
+ assert( pTerm->pExpr!=0 );
+ testcase( pTerm->wtFlags & TERM_VIRTUAL );
+ regRowid = sqlite3GetTempReg(pParse);
+ regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid);
+ sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter,
+ addrNxt, regRowid, 1);
+ VdbeCoverage(pParse->pVdbe);
+ }else{
+ u16 nEq = pLoop->u.btree.nEq;
+ int r1;
+ char *zStartAff;
+
+ assert( pLoop->wsFlags & WHERE_INDEXED );
+ r1 = codeAllEqualityTerms(pParse,pLevel,0,0,&zStartAff);
+ codeApplyAffinity(pParse, r1, nEq, zStartAff);
+ sqlite3DbFree(pParse->db, zStartAff);
+ sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter,
+ addrNxt, r1, nEq);
+ VdbeCoverage(pParse->pVdbe);
+ }
+ pLevel->regFilter = 0;
+ }
+}
+
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause
** implementation described by pWInfo.
@@ -1511,6 +1626,12 @@ Bitmask sqlite3WhereCodeOneLoopStart(
iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg);
if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg);
addrNxt = pLevel->addrNxt;
+ if( pLevel->regFilter ){
+ sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
+ iRowidReg, 1);
+ VdbeCoverage(v);
+ filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady);
+ }
sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
VdbeCoverage(v);
pLevel->op = OP_Noop;
@@ -1836,6 +1957,12 @@ Bitmask sqlite3WhereCodeOneLoopStart(
sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull);
VdbeComment((v, "NULL-scan pass ctr"));
}
+ if( pLevel->regFilter ){
+ sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
+ regBase, nEq);
+ VdbeCoverage(v);
+ filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady);
+ }
op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
assert( op!=0 );
diff --git a/test/join5.test b/test/join5.test
index e2ff2f6c27..0ae4ca1127 100644
--- a/test/join5.test
+++ b/test/join5.test
@@ -303,6 +303,7 @@ do_eqp_test 7.4 {
} {
QUERY PLAN
|--SCAN t3
+ |--BLOOM FILTER ON t4 (x=?)
`--SEARCH t4 USING INDEX t4xz (x=?)
}
diff --git a/tool/logest.c b/tool/logest.c
index e936e02cbe..580a3a6f96 100644
--- a/tool/logest.c
+++ b/tool/logest.c
@@ -75,6 +75,7 @@ static sqlite3_uint64 logEstToInt(LogEst x){
x /= 10;
if( n>=5 ) n -= 2;
else if( n>=1 ) n -= 1;
+ if( x>60 ) return ((sqlite3_uint64)0xffffffff)<<32 + 0xffffffff;
if( x>=3 ) return (n+8)<<(x-3);
return (n+8)>>(3-x);
}
@@ -149,7 +150,7 @@ int main(int argc, char **argv){
}else if( z[0]=='^' ){
a[n++] = (LogEst)atoi(z+1);
}else if( isInteger(z) ){
- a[n++] = logEstFromInteger(atoi(z));
+ a[n++] = logEstFromInteger(atoll(z));
}else if( isFloat(z) && z[0]!='-' ){
a[n++] = logEstFromDouble(atof(z));
}else{
@@ -161,6 +162,8 @@ int main(int argc, char **argv){
printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i]));
}else if( a[i]<10 ){
printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0);
+ }else if( a[i]>100 ){
+ printf("%5d (%lld)\n", a[i], logEstToInt(a[i]));
}else{
sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024;
printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100);