From 2db144c33b39985b159ec9210cc13bdb53c00e1c Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 1 Dec 2021 16:31:02 +0000
Subject: [PATCH 01/18] Add a Bloom filter to the automatic-index mechanism.
FossilOrigin-Name: 50ac4de1d7cbb586ea7969e1ae80ea8b021e194edc2fa7db19374b4ee9369bee
---
manifest | 27 ++++++------
manifest.uuid | 2 +-
src/sqliteInt.h | 1 +
src/vdbe.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++++
src/vdbeInt.h | 2 +-
src/vdbemem.c | 4 +-
src/where.c | 8 ++++
src/whereInt.h | 1 +
src/wherecode.c | 8 ++++
9 files changed, 143 insertions(+), 16 deletions(-)
diff --git a/manifest b/manifest
index 1799086100..8fae6a4e21 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sautomatic\sindex\sgenerator\slogic,\sbe\smore\sprecise\sabout\swhen\sa\npartial\sautomatic\sindex\sis\sallowed\sin\sorder\sto\scapture\smore\scases\swhere\sit\nis\slegal\sto\suse\sa\spartial\sautomatic\sindex.
-D 2021-11-30T14:07:58.372
+C Add\sa\sBloom\sfilter\sto\sthe\sautomatic-index\smechanism.
+D 2021-12-01T16:31:02.486
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -555,7 +555,7 @@ F src/shell.c.in 975f268ef261773fcbed1e519dfa10c4f33e8b1cffc12120563e61857fff07c
F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h 193e716a67c877a6054a8c261c932bdc64f8c8be9b66388c74f21d94a259b7ce
+F src/sqliteInt.h a44a3474b2247a82decc17644c68e00db5ba1bc239a6c005f6c08f6599083c01
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -622,13 +622,13 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c e98f1baf54a00db2c4669dbd04f8bbc89b5909a5b43e76fbbbf1a97007adba2b
+F src/vdbe.c a77525ddf690771b13db4114dce5305657efe6b7393920f2cbaa2447e8b83129
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
-F src/vdbeInt.h 31fbabdc1ed61d9695337dfe5269ea94e1cf615c17f5cafeaa1bb01066820bab
+F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
F src/vdbeaux.c 21db442d159fd745a7693d157b5f998260b6af4ca60de559fa3b7b68c7405af2
F src/vdbeblob.c 29c4118f7ee615cdee829e8401f6ead1b96b95d545b4de0042f6de39c962c652
-F src/vdbemem.c a3d91dc9bb9ef725db77e4e9de7e1acef43192c9f8406c307665d503e3c2837c
+F src/vdbemem.c da4d594084d581be6436582bb44bb128feeb138a3e6c313eda6749ebdc3a65ec
F src/vdbesort.c 513b481c8bab4a6578c92194a60cf3bc3b48736e4a53f8d2d7918121c5b594e7
F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823
F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c
@@ -637,9 +637,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 4b276017881185d0e1dc984df66c835cde6faf03834ef9ff34f48653f9144dec
-F src/whereInt.h 83877a75a1bce056ea44aff02f1dfa958ad1d6038c213ddadb8652003b45151d
-F src/wherecode.c 1f5b62f46d284c8886945eb7438415bc27e23e87bb60b9ee468fa6bd31268f33
+F src/where.c 1b8a6c53c34c59c765190484d245c457bedc211325de4c75a2070432fd67e314
+F src/whereInt.h 23f990791dc428f0bbb31f8ef5026b78b6290272de86185f4822c85f4c774803
+F src/wherecode.c 620e81077588a727876a86f87aab310c02fc4e4960691e48153d8a87ca1fa453
F src/whereexpr.c 17bdbf4f5b490e70a18635498f0b910a558f953a9bf80af7f19cbde6e60e6825
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 19c51b46e4095ee28badb10f4e08bbd330bda320c9a8806e93b8fc60ba211a2e
-R 0fbfb4a13255ae0bad405a99d2bdf9ec
+P 664b461bb5063d98047fc2e51a3827235cd9f55ca2e23cb66e719eac53fb5437
+R 157be6898a36d49d7a017b78b4cdbad6
+T *branch * bloom-filter
+T *sym-bloom-filter *
+T -sym-trunk *
U drh
-Z 511f59b4656ae7d0825d672340080bad
+Z 5c7d4c8ea12d463e648cb44a868717dc
diff --git a/manifest.uuid b/manifest.uuid
index 8a15f64b8a..2cbf640244 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-664b461bb5063d98047fc2e51a3827235cd9f55ca2e23cb66e719eac53fb5437
\ No newline at end of file
+50ac4de1d7cbb586ea7969e1ae80ea8b021e194edc2fa7db19374b4ee9369bee
\ No newline at end of file
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index b45151104a..d404b7b4fc 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1761,6 +1761,7 @@ struct sqlite3 {
#define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */
#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */
/* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */
+#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */
#define SQLITE_AllOpts 0xffffffff /* All optimizations */
/*
diff --git a/src/vdbe.c b/src/vdbe.c
index 3476c02daa..e38527356c 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -671,6 +671,35 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){
}
}
+/*
+** Default size of a bloom filter, in bytes
+*/
+#define SQLITE_BLOOM_SZ 10000
+
+/*
+** Compute a bloom filter hash using pOp->p4.i registers from aMem[] beginning
+** with pOp->p3. Return the hash.
+*/
+static unsigned int filterHash(const Mem *aMem, const Op *pOp){
+ int i, mx;
+ u32 h = 0;
+
+ i = pOp->p3;
+ assert( pOp->p4type==P4_INT32 );
+ mx = i + pOp->p4.i;
+ for(i=pOp->p3, mx=i+pOp->p4.i; i<mx; i++){
+ const Mem *p = &aMem[i];
+ if( p->flags & (MEM_Int|MEM_IntReal) ){
+ h += (u32)(p->u.i&0xffffffff);
+ }else if( p->flags & MEM_Real ){
+ h += (u32)(sqlite3VdbeIntValue(p)&0xffffffff);
+ }else if( p->flags & (MEM_Str|MEM_Blob) ){
+ h += p->n;
+ }
+ }
+ return h % (SQLITE_BLOOM_SZ*8);
+}
+
/*
** Return the symbolic name for the data type of a pMem
*/
@@ -8129,6 +8158,83 @@ case OP_Function: { /* group */
break;
}
+/* Opcode: FilterInit P1 * * * *
+** Synopsis: filter(P1) = empty
+**
+** Initialize register P1 so that is an empty bloom filter.
+*/
+case OP_FilterInit: {
+ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
+ pIn1 = &aMem[pOp->p1];
+ sqlite3VdbeMemSetZeroBlob(pIn1, SQLITE_BLOOM_SZ);
+ if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem;
+ break;
+}
+
+/* Opcode: FilterAdd P1 * P3 P4 *
+** Synopsis: filter(P1) += key(P3@P4)
+**
+** Compute a hash on the P4 registers starting with r[P3] and
+** add that hash to the bloom filter contained in r[P1].
+*/
+case OP_FilterAdd: {
+ u32 h;
+
+ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
+ pIn1 = &aMem[pOp->p1];
+ assert( pIn1->flags & MEM_Blob );
+ assert( pIn1->n==SQLITE_BLOOM_SZ );
+ h = filterHash(aMem, pOp);
+#ifdef SQLITE_DEBUG
+ if( db->flags&SQLITE_VdbeTrace ){
+ int ii;
+ for(ii=pOp->p3; ii<pOp->p3+pOp->p4.i; ii++){
+ registerTrace(ii, &aMem[ii]);
+ }
+ printf("hash = %u\n", h);
+ }
+#endif
+ assert( h>=0 && h<SQLITE_BLOOM_SZ*8 );
+ pIn1->z[h/8] |= 1<<(h&7);
+ break;
+}
+
+/* Opcode: Filter P1 P2 P3 P4 *
+** Synopsis: if key(P3@P4) not in filter(P1) goto P2
+**
+** Compute a hash on the key contained in the P4 registers starting
+** with r[P3]. Check to see if that hash is found in the
+** bloom filter hosted by register P1. If it is not present then
+** maybe jump to P2. Otherwise fall through.
+**
+** False negatives are harmless. It is always safe to fall through,
+** even if the value is in the bloom filter. A false negative causes
+** more CPU cycles to be used, but it should still yield the correct
+** answer. However, an incorrect answer may well arise from a
+** false positive - if the jump is taken when it should fall through.
+*/
+case OP_Filter: { /* jump */
+ u32 h;
+
+ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
+ pIn1 = &aMem[pOp->p1];
+ assert( pIn1->flags & MEM_Blob );
+ assert( pIn1->n==SQLITE_BLOOM_SZ );
+ h = filterHash(aMem, pOp);
+#ifdef SQLITE_DEBUG
+ if( db->flags&SQLITE_VdbeTrace ){
+ int ii;
+ for(ii=pOp->p3; ii<pOp->p3+pOp->p4.i; ii++){
+ registerTrace(ii, &aMem[ii]);
+ }
+ printf("hash = %u\n", h);
+ }
+#endif
+ assert( h>=0 && h<SQLITE_BLOOM_SZ*8 );
+ if( (pIn1->z[h/8] & (1<<(h&7)))==0 ) goto jump_to_p2;
+ break;
+}
+
/* Opcode: Trace P1 P2 * P4 *
**
** Write P4 on the statement trace output if statement tracing is
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 599d064165..c76cdbfdbc 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -538,7 +538,7 @@ int sqlite3VdbeMemSetRowSet(Mem*);
int sqlite3VdbeMemMakeWriteable(Mem*);
int sqlite3VdbeMemStringify(Mem*, u8, u8);
int sqlite3IntFloatCompare(i64,double);
-i64 sqlite3VdbeIntValue(Mem*);
+i64 sqlite3VdbeIntValue(const Mem*);
int sqlite3VdbeMemIntegerify(Mem*);
double sqlite3VdbeRealValue(Mem*);
int sqlite3VdbeBooleanValue(Mem*, int ifNull);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 570a2eb38c..5a9d15f465 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -596,12 +596,12 @@ static SQLITE_NOINLINE i64 doubleToInt64(double r){
**
** If pMem represents a string value, its encoding might be changed.
*/
-static SQLITE_NOINLINE i64 memIntValue(Mem *pMem){
+static SQLITE_NOINLINE i64 memIntValue(const Mem *pMem){
i64 value = 0;
sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc);
return value;
}
-i64 sqlite3VdbeIntValue(Mem *pMem){
+i64 sqlite3VdbeIntValue(const Mem *pMem){
int flags;
assert( pMem!=0 );
assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
diff --git a/src/where.c b/src/where.c
index 54e4ea8196..eaa45b0198 100644
--- a/src/where.c
+++ b/src/where.c
@@ -904,6 +904,10 @@ static void constructAutomaticIndex(
sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
VdbeComment((v, "for %s", pTable->zName));
+ if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){
+ pLevel->regFilter = ++pParse->nMem;
+ sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
+ }
/* Fill the automatic index with content */
pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom];
@@ -926,6 +930,10 @@ static void constructAutomaticIndex(
regBase = sqlite3GenerateIndexKey(
pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
);
+ if( pLevel->regFilter ){
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
+ regBase, pLoop->u.btree.nEq);
+ }
sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
diff --git a/src/whereInt.h b/src/whereInt.h
index f651e790cc..5b076e40e2 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -64,6 +64,7 @@ struct WhereLevel {
u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */
int addrLikeRep; /* LIKE range processing address */
#endif
+ int regFilter; /* Bloom filter */
u8 iFrom; /* Which entry in the FROM clause */
u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */
int p1, p2; /* Operands of the opcode used to end the loop */
diff --git a/src/wherecode.c b/src/wherecode.c
index 460ac4fe30..637db432ea 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1511,6 +1511,10 @@ Bitmask sqlite3WhereCodeOneLoopStart(
iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg);
if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg);
addrNxt = pLevel->addrNxt;
+ if( pLevel->regFilter ){
+ sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
+ iRowidReg, 1);
+ }
sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
VdbeCoverage(v);
pLevel->op = OP_Noop;
@@ -1836,6 +1840,10 @@ Bitmask sqlite3WhereCodeOneLoopStart(
sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull);
VdbeComment((v, "NULL-scan pass ctr"));
}
+ if( pLevel->regFilter ){
+ sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
+ regBase, nConstraint);
+ }
op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
assert( op!=0 );
From fa35f5c5a7868902019c6a18eeec52f4a577fc8b Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 4 Dec 2021 13:43:57 +0000
Subject: [PATCH 02/18] First attempt to use Bloom filters to optimize
star-schema queries.
FossilOrigin-Name: 28161fba9bcde5ae4b36b22d766c881b795af111a3a323c90f6149d0fea9297d
---
manifest | 14 +++---
manifest.uuid | 2 +-
src/where.c | 122 +++++++++++++++++++++++++++++++++++++++++++++++--
src/whereInt.h | 1 +
4 files changed, 126 insertions(+), 13 deletions(-)
diff --git a/manifest b/manifest
index 1e420b2cce..1e7e0a1ce4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\ssqlite3WhereBegin()\ssimplification\sfrom\strunk.
-D 2021-12-03T19:10:17.141
+C First\sattempt\sto\suse\sBloom\sfilters\sto\soptimize\sstar-schema\squeries.
+D 2021-12-04T13:43:57.343
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -637,8 +637,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c a0787ad0a3447685e553725dab50e52c9763b5f10e2506255713dbd736c89998
-F src/whereInt.h 83ae6f7d0fce8b5164d97698790e0005a909b3d06d73df86a4a6bb0d9411861e
+F src/where.c e9cfeae040b60c21244f40fed1a3473f048ee20cb2fb987d5028350e4bead886
+F src/whereInt.h 2c9d149b1b41c59f977deb1882e95632800946fcf15c83eaffb227afece04e51
F src/wherecode.c 620e81077588a727876a86f87aab310c02fc4e4960691e48153d8a87ca1fa453
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 11d97fb8be6b5155f38df130d7e828edd0f381e32f651458939b1cb9cb973fff 6225e9abcb0261fefca4a26530524ffc449f937f8ae1ece718af2c3c3d73d78d
-R 91b705f2730ed0122ffcaf8efbe0af73
+P 41ba2dfdd3a18671fc78d60935a16fa50f36af3d6481eff2ca9fba88e7093997
+R a2ed82c3d908c8671ab026e425f8c4f5
U drh
-Z dab48a2f628412943ed7bfde21b0a086
+Z 92e3ffa168d28fa3c757a00a1d8afcc3
diff --git a/manifest.uuid b/manifest.uuid
index 6adb3fed23..383563aee3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-41ba2dfdd3a18671fc78d60935a16fa50f36af3d6481eff2ca9fba88e7093997
\ No newline at end of file
+28161fba9bcde5ae4b36b22d766c881b795af111a3a323c90f6149d0fea9297d
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 371bf52305..22ed0c9b5b 100644
--- a/src/where.c
+++ b/src/where.c
@@ -750,7 +750,7 @@ static int termCanDriveIndex(
** and to set up the WhereLevel object pLevel so that the code generator
** makes use of the automatic index.
*/
-static void constructAutomaticIndex(
+static SQLITE_NOINLINE void constructAutomaticIndex(
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
SrcItem *pSrc, /* The FROM clause term to get the next index */
@@ -965,6 +965,62 @@ end_auto_index_create:
}
#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
+/*
+** Create a Bloom filter for the WhereLevel in the parameter.
+*/
+static SQLITE_NOINLINE void constructBloomFilter(
+ WhereInfo *pWInfo, /* The WHERE clause */
+ WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
+){
+ int addrTop;
+ int addrCont;
+ WhereTerm *pTerm;
+ WhereTerm *pWCEnd;
+ Parse *pParse = pWInfo->pParse;
+ Vdbe *v = pParse->pVdbe;
+ WhereLoop *pLoop = pLevel->pWLoop;
+ int iCur;
+
+
+ assert( pLoop!=0 );
+ assert( v!=0 );
+ iCur = pLevel->iTabCur;
+ addrCont = sqlite3VdbeMakeLabel(pParse);
+ addrTop = sqlite3VdbeAddOp0(v, OP_Once);
+ pLevel->regFilter = ++pParse->nMem;
+ sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
+ sqlite3VdbeAddOp1(v, OP_Rewind, iCur);
+ pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
+ for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
+ Expr *pExpr = pTerm->pExpr;
+ if( (pTerm->wtFlags & TERM_VIRTUAL)==0
+ && sqlite3ExprIsTableConstant(pExpr, iCur)
+ ){
+ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
+ }
+ }
+ if( pLoop->wsFlags & WHERE_IPK ){
+ int r1 = sqlite3GetTempReg(pParse);
+ sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
+ }else{
+ Index *pIdx = pLoop->u.btree.pIndex;
+ int r1 = sqlite3GetTempRange(pParse, pIdx->nKeyCol);
+ int n = pIdx->nKeyCol;
+ int jj;
+ for(jj=0; jj<n; jj++){
+ int iCol = pIdx->aiColumn[jj];
+ sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
+ }
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
+ }
+ sqlite3VdbeResolveLabel(v, addrCont);
+ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
+ sqlite3VdbeJumpHere(v, addrTop);
+ sqlite3VdbeJumpHere(v, addrTop+2);
+}
+
+
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Allocate and populate an sqlite3_index_info structure. It is the
@@ -4841,6 +4897,49 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
return notReady;
}
+/*
+** Check to see if there are any SEARCH loops that might benefit from
+** using a Bloom filter. Consider a Bloom filter if:
+**
+** (1) The SEARCH happens more than N times where N is the number
+** of rows in the table that is being considered for the Bloom
+** filter. (TO DO: Make this condition more precise.)
+** (2) Most searches are expected to find zero rows
+** (3) The table being searched is not the right table of a LEFT JOIN
+** (4) Bloom-filter processing is not disabled
+**
+** This block of code merely checks to see if a Bloom filter would be
+** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
+** WhereLoop. The implementation of the Bloom filter comes further
+** down where the code for each WhereLoop is generated.
+*/
+static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
+ WhereInfo *pWInfo,
+ sqlite3 *db
+){
+ int i;
+ LogEst nSearch;
+ SrcItem *pItem;
+
+ assert( pWInfo->nLevel>=2 );
+ assert( OptimizationEnabled(db, SQLITE_BloomFilter) );
+ nSearch = pWInfo->a[0].pWLoop->nOut;
+ for(i=1; i<pWInfo->nLevel; i++){
+ WhereLoop *pLoop = pWInfo->a[i].pWLoop;
+ if( (pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0
+ && (pLoop->wsFlags & WHERE_COLUMN_EQ)!=0
+ && pLoop->nOut<0
+ && nSearch > (pItem = &pWInfo->pTabList->a[pLoop->iTab])->pTab->nRowLogEst
+ && (pItem->fg.jointype & JT_LEFT)==0
+ ){
+ pLoop->wsFlags |= WHERE_BLOOMFILTER;
+ pLoop->wsFlags &= ~WHERE_IDX_ONLY;
+ WHERETRACE(0xffff, ("-> use Bloom-filter on loop %c\n", pLoop->cId));
+ }
+ nSearch += pLoop->nOut;
+ }
+}
+
/*
** Generate the beginning of the loop used for WHERE clause processing.
** The return value is a pointer to an opaque structure that contains
@@ -5230,6 +5329,15 @@ WhereInfo *sqlite3WhereBegin(
assert( nTabList>0 );
}
+ /* Check to see if there are any SEARCH loops that might benefit from
+ ** using a Bloom filter.
+ */
+ if( pWInfo->nLevel>=2
+ && OptimizationEnabled(db, SQLITE_BloomFilter)
+ ){
+ whereCheckIfBloomFilterIsUseful(pWInfo, db);
+ }
+
#if defined(WHERETRACE_ENABLED)
if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
@@ -5418,13 +5526,17 @@ WhereInfo *sqlite3WhereBegin(
if( pParse->nErr ) goto whereBeginError;
pLevel = &pWInfo->a[ii];
wsFlags = pLevel->pWLoop->wsFlags;
+ if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
+ if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
- if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){
- constructAutomaticIndex(pParse, &pWInfo->sWC,
- &pTabList->a[pLevel->iFrom], notReady, pLevel);
+ constructAutomaticIndex(pParse, &pWInfo->sWC,
+ &pTabList->a[pLevel->iFrom], notReady, pLevel);
+#endif
+ }else{
+ constructBloomFilter(pWInfo, pLevel);
+ }
if( db->mallocFailed ) goto whereBeginError;
}
-#endif
addrExplain = sqlite3WhereExplainOneScan(
pParse, pTabList, pLevel, wctrlFlags
);
diff --git a/src/whereInt.h b/src/whereInt.h
index ff6ac4c397..fbe84787f9 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -600,5 +600,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
#define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */
#define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */
#define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */
+#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */
#endif /* !defined(SQLITE_WHEREINT_H) */
From 19ce9aafdb200670d5ee9c3f1b139801a988f2ba Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 4 Dec 2021 13:52:08 +0000
Subject: [PATCH 03/18] Add the "WITH BLOOM FILTER" clause to the EXPLAIN QUERY
PLAN output for cases were a Bloom filter is used.
FossilOrigin-Name: 8e078c0e404fe4b3661dd4a11a992a8914c99e2a144cecc417421cbd68fa08cc
---
manifest | 12 ++++++------
manifest.uuid | 2 +-
src/wherecode.c | 3 +++
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/manifest b/manifest
index 1e7e0a1ce4..dfe53bde8f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C First\sattempt\sto\suse\sBloom\sfilters\sto\soptimize\sstar-schema\squeries.
-D 2021-12-04T13:43:57.343
+C Add\sthe\s"WITH\sBLOOM\sFILTER"\sclause\sto\sthe\sEXPLAIN\sQUERY\sPLAN\soutput\sfor\ncases\swere\sa\sBloom\sfilter\sis\sused.
+D 2021-12-04T13:52:08.699
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -639,7 +639,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
F src/where.c e9cfeae040b60c21244f40fed1a3473f048ee20cb2fb987d5028350e4bead886
F src/whereInt.h 2c9d149b1b41c59f977deb1882e95632800946fcf15c83eaffb227afece04e51
-F src/wherecode.c 620e81077588a727876a86f87aab310c02fc4e4960691e48153d8a87ca1fa453
+F src/wherecode.c 26b0cbc985deafe702037ed00bcf40857d129be0d80d198ac87644777b59984d
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 41ba2dfdd3a18671fc78d60935a16fa50f36af3d6481eff2ca9fba88e7093997
-R a2ed82c3d908c8671ab026e425f8c4f5
+P 28161fba9bcde5ae4b36b22d766c881b795af111a3a323c90f6149d0fea9297d
+R 6eeaa3a323a583fb23be16ffd26ce73a
U drh
-Z 92e3ffa168d28fa3c757a00a1d8afcc3
+Z a160cdc5e20e6c677a6900e142dccac2
diff --git a/manifest.uuid b/manifest.uuid
index 383563aee3..030f03de8f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-28161fba9bcde5ae4b36b22d766c881b795af111a3a323c90f6149d0fea9297d
\ No newline at end of file
+8e078c0e404fe4b3661dd4a11a992a8914c99e2a144cecc417421cbd68fa08cc
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index 637db432ea..2e616ad592 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -196,6 +196,9 @@ int sqlite3WhereExplainOneScan(
pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
}
#endif
+ if( flags & WHERE_BLOOMFILTER ){
+ sqlite3_str_appendf(&str, " WITH BLOOM FILTER");
+ }
#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS
if( pLoop->nOut>=10 ){
sqlite3_str_appendf(&str, " (~%llu rows)",
From 770dade262f88ec531d7708ac2d0f2e7882dcb16 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 4 Dec 2021 14:24:30 +0000
Subject: [PATCH 04/18] Apply the Bloom filter only on those terms of an index
that have equality constraints.
FossilOrigin-Name: a70429596a3c6a413b03118b0d800521b3526d99dcf88a48acc3189b51518d82
---
manifest | 14 +++++++-------
manifest.uuid | 2 +-
src/where.c | 2 +-
src/wherecode.c | 2 +-
4 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/manifest b/manifest
index dfe53bde8f..f576dec37b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"WITH\sBLOOM\sFILTER"\sclause\sto\sthe\sEXPLAIN\sQUERY\sPLAN\soutput\sfor\ncases\swere\sa\sBloom\sfilter\sis\sused.
-D 2021-12-04T13:52:08.699
+C Apply\sthe\sBloom\sfilter\sonly\son\sthose\sterms\sof\san\sindex\sthat\shave\sequality\nconstraints.
+D 2021-12-04T14:24:30.524
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -637,9 +637,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c e9cfeae040b60c21244f40fed1a3473f048ee20cb2fb987d5028350e4bead886
+F src/where.c d203a7acda584940eee49d80e38067ba354410ca08782792bdf4d6f56ca20aa3
F src/whereInt.h 2c9d149b1b41c59f977deb1882e95632800946fcf15c83eaffb227afece04e51
-F src/wherecode.c 26b0cbc985deafe702037ed00bcf40857d129be0d80d198ac87644777b59984d
+F src/wherecode.c ab83c1f7c593ab5e9268c51dde80f478e48468e8c1e25866a4f445e7b4282b78
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 28161fba9bcde5ae4b36b22d766c881b795af111a3a323c90f6149d0fea9297d
-R 6eeaa3a323a583fb23be16ffd26ce73a
+P 8e078c0e404fe4b3661dd4a11a992a8914c99e2a144cecc417421cbd68fa08cc
+R a78591e6ea873c8de7a64c271c45d204
U drh
-Z a160cdc5e20e6c677a6900e142dccac2
+Z 73637cbd9e62c30bcdc98bccabee0c03
diff --git a/manifest.uuid b/manifest.uuid
index 030f03de8f..23eed8d9fa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8e078c0e404fe4b3661dd4a11a992a8914c99e2a144cecc417421cbd68fa08cc
\ No newline at end of file
+a70429596a3c6a413b03118b0d800521b3526d99dcf88a48acc3189b51518d82
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 22ed0c9b5b..87a00ef200 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1006,7 +1006,7 @@ static SQLITE_NOINLINE void constructBloomFilter(
}else{
Index *pIdx = pLoop->u.btree.pIndex;
int r1 = sqlite3GetTempRange(pParse, pIdx->nKeyCol);
- int n = pIdx->nKeyCol;
+ int n = pLoop->u.btree.nEq;
int jj;
for(jj=0; jj<n; jj++){
int iCol = pIdx->aiColumn[jj];
diff --git a/src/wherecode.c b/src/wherecode.c
index 2e616ad592..fbb40ad4e3 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1845,7 +1845,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
}
if( pLevel->regFilter ){
sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
- regBase, nConstraint);
+ regBase, nEq);
}
op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
From 067c60cfc9aa3ec493f3bdd8f62d225846743a2a Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 4 Dec 2021 18:45:08 +0000
Subject: [PATCH 05/18] Add VdbeCoverage() macros. Adjust the Bloom-filter
hash function so that it correctly deals with zero-blobs.
FossilOrigin-Name: 629ee2e3e3125bfd2af435c6713d49e46691213ad15db0a5e93a63a77f1130c2
---
manifest | 16 ++++++++--------
manifest.uuid | 2 +-
src/vdbe.c | 8 +++++++-
src/where.c | 5 +++--
src/wherecode.c | 2 ++
5 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/manifest b/manifest
index f576dec37b..2c58aedeeb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Apply\sthe\sBloom\sfilter\sonly\son\sthose\sterms\sof\san\sindex\sthat\shave\sequality\nconstraints.
-D 2021-12-04T14:24:30.524
+C Add\sVdbeCoverage()\smacros.\s\sAdjust\sthe\sBloom-filter\shash\sfunction\sso\sthat\sit\ncorrectly\sdeals\swith\szero-blobs.
+D 2021-12-04T18:45:08.377
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -622,7 +622,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c d4435e88e8abb076c955ee8262f548afcba77daa28b75f937ed914404bb29be7
+F src/vdbe.c 94af4eba93ad9ca7dd929cd19792ce2a5feb4797a7a64ec3cb3b2277e1467a8b
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
@@ -637,9 +637,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c d203a7acda584940eee49d80e38067ba354410ca08782792bdf4d6f56ca20aa3
+F src/where.c 2607f8008b27f03003d62ed3b90e1818fda1dd5727daf020c8b3e526aff57269
F src/whereInt.h 2c9d149b1b41c59f977deb1882e95632800946fcf15c83eaffb227afece04e51
-F src/wherecode.c ab83c1f7c593ab5e9268c51dde80f478e48468e8c1e25866a4f445e7b4282b78
+F src/wherecode.c 2253f91bcded8932a4211238fa7dcf6ab40ff68dbd4c2d88eb4411f12b19552c
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8e078c0e404fe4b3661dd4a11a992a8914c99e2a144cecc417421cbd68fa08cc
-R a78591e6ea873c8de7a64c271c45d204
+P a70429596a3c6a413b03118b0d800521b3526d99dcf88a48acc3189b51518d82
+R f0b7f8e349624b23e895cc40d5e34cdd
U drh
-Z 73637cbd9e62c30bcdc98bccabee0c03
+Z 186d2bb7a1014e61ff9f3f3c0fb31234
diff --git a/manifest.uuid b/manifest.uuid
index 23eed8d9fa..55275e45b8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a70429596a3c6a413b03118b0d800521b3526d99dcf88a48acc3189b51518d82
\ No newline at end of file
+629ee2e3e3125bfd2af435c6713d49e46691213ad15db0a5e93a63a77f1130c2
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 9b6a4106d3..acbbee892f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -695,6 +695,7 @@ static unsigned int filterHash(const Mem *aMem, const Op *pOp){
h += (u32)(sqlite3VdbeIntValue(p)&0xffffffff);
}else if( p->flags & (MEM_Str|MEM_Blob) ){
h += p->n;
+ if( p->flags & MEM_Zero ) h += p->u.nZero;
}
}
return h % (SQLITE_BLOOM_SZ*8);
@@ -8229,7 +8230,12 @@ case OP_Filter: { /* jump */
}
#endif
assert( h>=0 && h<SQLITE_BLOOM_SZ*8 );
- if( (pIn1->z[h/8] & (1<<(h&7)))==0 ) goto jump_to_p2;
+ if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){
+ VdbeBranchTaken(1, 2);
+ goto jump_to_p2;
+ }else{
+ VdbeBranchTaken(0, 2);
+ }
break;
}
diff --git a/src/where.c b/src/where.c
index 87a00ef200..0d205125c8 100644
--- a/src/where.c
+++ b/src/where.c
@@ -986,10 +986,10 @@ static SQLITE_NOINLINE void constructBloomFilter(
assert( v!=0 );
iCur = pLevel->iTabCur;
addrCont = sqlite3VdbeMakeLabel(pParse);
- addrTop = sqlite3VdbeAddOp0(v, OP_Once);
+ addrTop = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
pLevel->regFilter = ++pParse->nMem;
sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
- sqlite3VdbeAddOp1(v, OP_Rewind, iCur);
+ sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
Expr *pExpr = pTerm->pExpr;
@@ -1016,6 +1016,7 @@ static SQLITE_NOINLINE void constructBloomFilter(
}
sqlite3VdbeResolveLabel(v, addrCont);
sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
+ VdbeCoverage(v);
sqlite3VdbeJumpHere(v, addrTop);
sqlite3VdbeJumpHere(v, addrTop+2);
}
diff --git a/src/wherecode.c b/src/wherecode.c
index fbb40ad4e3..e02a736428 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1517,6 +1517,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
if( pLevel->regFilter ){
sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
iRowidReg, 1);
+ VdbeCoverage(v);
}
sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
VdbeCoverage(v);
@@ -1846,6 +1847,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
if( pLevel->regFilter ){
sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
regBase, nEq);
+ VdbeCoverage(v);
}
op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
From fecbf0a179ae9c2aae0fbca17c80ce776ef78704 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sat, 4 Dec 2021 21:11:18 +0000
Subject: [PATCH 06/18] Miscellaneous cleanup of the new Bloom-filter code.
FossilOrigin-Name: 201b6dd875b0ae2bbc9969b098e88abfc09e37b59e857decd41f2dcbeeb13e01
---
manifest | 12 ++++++------
manifest.uuid | 2 +-
src/where.c | 31 +++++++++++++++----------------
3 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/manifest b/manifest
index 2c58aedeeb..e1808eea5b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sVdbeCoverage()\smacros.\s\sAdjust\sthe\sBloom-filter\shash\sfunction\sso\sthat\sit\ncorrectly\sdeals\swith\szero-blobs.
-D 2021-12-04T18:45:08.377
+C Miscellaneous\scleanup\sof\sthe\snew\sBloom-filter\scode.
+D 2021-12-04T21:11:18.037
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -637,7 +637,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 2607f8008b27f03003d62ed3b90e1818fda1dd5727daf020c8b3e526aff57269
+F src/where.c 957543456a7d1683d21ed8c890bcdaa6414716082dfb076bb43decdc0f87909e
F src/whereInt.h 2c9d149b1b41c59f977deb1882e95632800946fcf15c83eaffb227afece04e51
F src/wherecode.c 2253f91bcded8932a4211238fa7dcf6ab40ff68dbd4c2d88eb4411f12b19552c
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a70429596a3c6a413b03118b0d800521b3526d99dcf88a48acc3189b51518d82
-R f0b7f8e349624b23e895cc40d5e34cdd
+P 629ee2e3e3125bfd2af435c6713d49e46691213ad15db0a5e93a63a77f1130c2
+R 5366235cd370e2a6adf5aabc2d25d6ed
U drh
-Z 186d2bb7a1014e61ff9f3f3c0fb31234
+Z ab3ac181064c99ca5c8ac656f604bf60
diff --git a/manifest.uuid b/manifest.uuid
index 55275e45b8..5fe0ded6ae 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-629ee2e3e3125bfd2af435c6713d49e46691213ad15db0a5e93a63a77f1130c2
\ No newline at end of file
+201b6dd875b0ae2bbc9969b098e88abfc09e37b59e857decd41f2dcbeeb13e01
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 0d205125c8..c1a538dcc9 100644
--- a/src/where.c
+++ b/src/where.c
@@ -717,9 +717,9 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
** index existed.
*/
static int termCanDriveIndex(
- WhereTerm *pTerm, /* WHERE clause term to check */
- SrcItem *pSrc, /* Table we are trying to access */
- Bitmask notReady /* Tables in outer loops of the join */
+ const WhereTerm *pTerm, /* WHERE clause term to check */
+ const SrcItem *pSrc, /* Table we are trying to access */
+ const Bitmask notReady /* Tables in outer loops of the join */
){
char aff;
if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
@@ -752,9 +752,9 @@ static int termCanDriveIndex(
*/
static SQLITE_NOINLINE void constructAutomaticIndex(
Parse *pParse, /* The parsing context */
- WhereClause *pWC, /* The WHERE clause */
- SrcItem *pSrc, /* The FROM clause term to get the next index */
- Bitmask notReady, /* Mask of cursors that are not available */
+ const WhereClause *pWC, /* The WHERE clause */
+ const SrcItem *pSrc, /* The FROM clause term to get the next index */
+ const Bitmask notReady, /* Mask of cursors that are not available */
WhereLevel *pLevel /* Write new index here */
){
int nKeyCol; /* Number of columns in the constructed index */
@@ -969,16 +969,16 @@ end_auto_index_create:
** Create a Bloom filter for the WhereLevel in the parameter.
*/
static SQLITE_NOINLINE void constructBloomFilter(
- WhereInfo *pWInfo, /* The WHERE clause */
+ const WhereInfo *pWInfo, /* The WHERE clause */
WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
){
int addrTop;
int addrCont;
- WhereTerm *pTerm;
- WhereTerm *pWCEnd;
+ const WhereTerm *pTerm;
+ const WhereTerm *pWCEnd;
Parse *pParse = pWInfo->pParse;
Vdbe *v = pParse->pVdbe;
- WhereLoop *pLoop = pLevel->pWLoop;
+ const WhereLoop *pLoop = pLevel->pWLoop;
int iCur;
@@ -4915,21 +4915,20 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
** down where the code for each WhereLoop is generated.
*/
static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
- WhereInfo *pWInfo,
- sqlite3 *db
+ const WhereInfo *pWInfo
){
int i;
LogEst nSearch;
SrcItem *pItem;
assert( pWInfo->nLevel>=2 );
- assert( OptimizationEnabled(db, SQLITE_BloomFilter) );
+ assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
nSearch = pWInfo->a[0].pWLoop->nOut;
for(i=1; i<pWInfo->nLevel; i++){
WhereLoop *pLoop = pWInfo->a[i].pWLoop;
- if( (pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0
+ if( pLoop->nOut<0
+ && (pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0
&& (pLoop->wsFlags & WHERE_COLUMN_EQ)!=0
- && pLoop->nOut<0
&& nSearch > (pItem = &pWInfo->pTabList->a[pLoop->iTab])->pTab->nRowLogEst
&& (pItem->fg.jointype & JT_LEFT)==0
){
@@ -5336,7 +5335,7 @@ WhereInfo *sqlite3WhereBegin(
if( pWInfo->nLevel>=2
&& OptimizationEnabled(db, SQLITE_BloomFilter)
){
- whereCheckIfBloomFilterIsUseful(pWInfo, db);
+ whereCheckIfBloomFilterIsUseful(pWInfo);
}
#if defined(WHERETRACE_ENABLED)
From 35685d3e5e0e28a7631616809be450b090029482 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sun, 5 Dec 2021 00:45:55 +0000
Subject: [PATCH 07/18] Try to run all Bloom filters before any Seeks. This
gives a small performance gain on the Star-Schema Benchmark.
FossilOrigin-Name: 5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
---
manifest | 16 +++++++-------
manifest.uuid | 2 +-
src/where.c | 8 ++++---
src/whereInt.h | 1 +
src/wherecode.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 71 insertions(+), 12 deletions(-)
diff --git a/manifest b/manifest
index e1808eea5b..1a9a4b84ff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Miscellaneous\scleanup\sof\sthe\snew\sBloom-filter\scode.
-D 2021-12-04T21:11:18.037
+C Try\sto\srun\sall\sBloom\sfilters\sbefore\sany\sSeeks.\s\sThis\sgives\sa\ssmall\sperformance\ngain\son\sthe\sStar-Schema\sBenchmark.
+D 2021-12-05T00:45:55.528
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -637,9 +637,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 957543456a7d1683d21ed8c890bcdaa6414716082dfb076bb43decdc0f87909e
-F src/whereInt.h 2c9d149b1b41c59f977deb1882e95632800946fcf15c83eaffb227afece04e51
-F src/wherecode.c 2253f91bcded8932a4211238fa7dcf6ab40ff68dbd4c2d88eb4411f12b19552c
+F src/where.c 3d29b27e345a28f76ecb20b0c789a224e9dc74534363c3ddbc44c283eb4cb708
+F src/whereInt.h 3c634e184de128d99c2b017f899f2309c77330de54193ff970597e9acdeaff3b
+F src/wherecode.c 48b456c910d552e1edf098071546c523837356ba0e3703d7b8fd9fc819da1c9b
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 629ee2e3e3125bfd2af435c6713d49e46691213ad15db0a5e93a63a77f1130c2
-R 5366235cd370e2a6adf5aabc2d25d6ed
+P 201b6dd875b0ae2bbc9969b098e88abfc09e37b59e857decd41f2dcbeeb13e01
+R 8ad2d75d69a2ffc06bdf1b52eb7537e1
U drh
-Z ab3ac181064c99ca5c8ac656f604bf60
+Z 42400fd1219d022ab29b577e6d964b89
diff --git a/manifest.uuid b/manifest.uuid
index 5fe0ded6ae..cca08a44eb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-201b6dd875b0ae2bbc9969b098e88abfc09e37b59e857decd41f2dcbeeb13e01
\ No newline at end of file
+5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index c1a538dcc9..2f7eb06097 100644
--- a/src/where.c
+++ b/src/where.c
@@ -968,7 +968,7 @@ end_auto_index_create:
/*
** Create a Bloom filter for the WhereLevel in the parameter.
*/
-static SQLITE_NOINLINE void constructBloomFilter(
+SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
const WhereInfo *pWInfo, /* The WHERE clause */
WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
){
@@ -1003,16 +1003,18 @@ static SQLITE_NOINLINE void constructBloomFilter(
int r1 = sqlite3GetTempReg(pParse);
sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
+ sqlite3ReleaseTempReg(pParse, r1);
}else{
Index *pIdx = pLoop->u.btree.pIndex;
- int r1 = sqlite3GetTempRange(pParse, pIdx->nKeyCol);
int n = pLoop->u.btree.nEq;
+ int r1 = sqlite3GetTempRange(pParse, n);
int jj;
for(jj=0; jj<n; jj++){
int iCol = pIdx->aiColumn[jj];
sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
}
sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
+ sqlite3ReleaseTempRange(pParse, r1, n);
}
sqlite3VdbeResolveLabel(v, addrCont);
sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
@@ -5533,7 +5535,7 @@ WhereInfo *sqlite3WhereBegin(
&pTabList->a[pLevel->iFrom], notReady, pLevel);
#endif
}else{
- constructBloomFilter(pWInfo, pLevel);
+ sqlite3ConstructBloomFilter(pWInfo, pLevel);
}
if( db->mallocFailed ) goto whereBeginError;
}
diff --git a/src/whereInt.h b/src/whereInt.h
index fbe84787f9..19261ca397 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -484,6 +484,7 @@ struct WhereInfo {
** where.c:
*/
Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
+void sqlite3ConstructBloomFilter(const WhereInfo*, WhereLevel*);
#ifdef WHERETRACE_ENABLED
void sqlite3WhereClausePrint(WhereClause *pWC);
void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm);
diff --git a/src/wherecode.c b/src/wherecode.c
index e02a736428..adc7d5b8e1 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1304,6 +1304,60 @@ static void whereApplyPartialIndexConstraints(
}
}
+#if 1
+/*
+** An OP_Filter has just been generated, but the corresponding
+** index search has not yet been performed. This routine
+** checks to see if there are additional WHERE_BLOOMFILTER in
+** inner loops that can be evaluated right away, and if there are,
+** it evaluates those filters as well, and removes the WHERE_BLOOMFILTER
+** tag.
+*/
+static SQLITE_NOINLINE void filterPullDown(
+ Parse *pParse, /* Parsing context */
+ WhereInfo *pWInfo, /* Complete information about the WHERE clause */
+ int iLevel, /* Which level of pWInfo->a[] should be coded */
+ int addrNxt, /* Jump here to bypass inner loops */
+ Bitmask notReady /* Loops that are not ready */
+){
+ while( ++iLevel < pWInfo->nLevel ){
+ WhereLevel *pLevel = &pWInfo->a[iLevel];
+ WhereLoop *pLoop = pLevel->pWLoop;
+ if( (pLoop->wsFlags & WHERE_BLOOMFILTER)==0 ) continue;
+ if( pLoop->prereq & notReady ) continue;
+ sqlite3ConstructBloomFilter(pWInfo, &pWInfo->a[iLevel]);
+ if( pLoop->wsFlags & WHERE_IPK ){
+ WhereTerm *pTerm = pLoop->aLTerm[0];
+ int r1, regRowid;
+ assert( pTerm!=0 );
+ assert( pTerm->pExpr!=0 );
+ testcase( pTerm->wtFlags & TERM_VIRTUAL );
+ r1 = sqlite3GetTempReg(pParse);
+ regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, r1);
+ if( regRowid!=r1 ) sqlite3ReleaseTempReg(pParse, r1);
+ sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter,
+ addrNxt, regRowid, 1);
+ VdbeCoverage(pParse->pVdbe);
+ }else{
+ u16 nEq = pLoop->u.btree.nEq;
+ int r1;
+ char *zStartAff;
+
+ assert( pLoop->wsFlags & WHERE_INDEXED );
+ r1 = codeAllEqualityTerms(pParse,pLevel,0,0,&zStartAff);
+ codeApplyAffinity(pParse, r1, nEq, zStartAff);
+ sqlite3DbFree(pParse->db, zStartAff);
+ sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter,
+ addrNxt, r1, nEq);
+ VdbeCoverage(pParse->pVdbe);
+ }
+ pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
+ }
+}
+#else
+#define filterPullDown(A,B,C,D,E)
+#endif
+
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause
** implementation described by pWInfo.
@@ -1518,6 +1572,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
iRowidReg, 1);
VdbeCoverage(v);
+ filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady);
}
sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
VdbeCoverage(v);
@@ -1848,6 +1903,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
regBase, nEq);
VdbeCoverage(v);
+ filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady);
}
op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
From 6ae49e67cc6e40538d1de2e0239dbdd59e487814 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Sun, 5 Dec 2021 20:19:47 +0000
Subject: [PATCH 08/18] Run as many Bloom filters as possible before index
lookups.
FossilOrigin-Name: 06f6fefd67086896bc49272c6319545ff6c6792f18babe23aced27b60b032119
---
manifest | 18 +++----
manifest.uuid | 2 +-
src/sqliteInt.h | 1 +
src/where.c | 129 ++++++++++++++++++++++++++++++------------------
src/whereInt.h | 7 ++-
src/wherecode.c | 77 +++++++++++++++++++++++------
6 files changed, 158 insertions(+), 76 deletions(-)
diff --git a/manifest b/manifest
index 1a9a4b84ff..47992d2a2e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Try\sto\srun\sall\sBloom\sfilters\sbefore\sany\sSeeks.\s\sThis\sgives\sa\ssmall\sperformance\ngain\son\sthe\sStar-Schema\sBenchmark.
-D 2021-12-05T00:45:55.528
+C Run\sas\smany\sBloom\sfilters\sas\spossible\sbefore\sindex\slookups.
+D 2021-12-05T20:19:47.744
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -555,7 +555,7 @@ F src/shell.c.in e7ee6517544d075d9f06ee2571567026b89cf9fbeef16a74918019b1cb42576
F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h 4a19ff088cc3bb1467cca94011eaf7e8432476124bbb72c77053d9f1e6b0a6c7
+F src/sqliteInt.h 178eb899c1edc08dcddf37e79dfaa39404a1f5d44a1d512509cd5d41867aa836
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -637,9 +637,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 3d29b27e345a28f76ecb20b0c789a224e9dc74534363c3ddbc44c283eb4cb708
-F src/whereInt.h 3c634e184de128d99c2b017f899f2309c77330de54193ff970597e9acdeaff3b
-F src/wherecode.c 48b456c910d552e1edf098071546c523837356ba0e3703d7b8fd9fc819da1c9b
+F src/where.c b07c5eefecffa1b69b91c366a83c69d01a83f1c900b9d9b1ffb6eb5ab59902a1
+F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
+F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 201b6dd875b0ae2bbc9969b098e88abfc09e37b59e857decd41f2dcbeeb13e01
-R 8ad2d75d69a2ffc06bdf1b52eb7537e1
+P 5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
+R da3cb867f9ab99abba060de93457c8d9
U drh
-Z 42400fd1219d022ab29b577e6d964b89
+Z d87a509afa829d1cf21b5a6dcadef441
diff --git a/manifest.uuid b/manifest.uuid
index cca08a44eb..669a897f37 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
\ No newline at end of file
+06f6fefd67086896bc49272c6319545ff6c6792f18babe23aced27b60b032119
\ No newline at end of file
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index e96947e81d..896b2aa422 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1762,6 +1762,7 @@ struct sqlite3 {
#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */
/* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */
#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */
+#define SQLITE_BloomPulldown 0x00100000 /* Run Bloom filters early */
#define SQLITE_AllOpts 0xffffffff /* All optimizations */
/*
diff --git a/src/where.c b/src/where.c
index 2f7eb06097..1d88c3ed30 100644
--- a/src/where.c
+++ b/src/where.c
@@ -966,61 +966,92 @@ end_auto_index_create:
#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
/*
-** Create a Bloom filter for the WhereLevel in the parameter.
+** Generate bytecode that will initialize a Bloom filter that is appropriate
+** for pLevel.
+**
+** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
+** flag set, initialize a Bloomfilter for them as well. Except don't do
+** this recursive initialization if the SQLITE_BloomPulldown optimization has
+** been turned off.
+**
+** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
+** from the loop, but the regFilter value is set to a register that implements
+** the Bloom filter. When regFilter is positive, the
+** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
+** and skip the subsequence B-Tree seek if the Bloom filter indicates that
+** no matching rows exist.
+**
+** This routine may only be called if it has previously been determined that
+** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
+** is set.
*/
-SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
- const WhereInfo *pWInfo, /* The WHERE clause */
- WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
+static SQLITE_NOINLINE void constructBloomFilter(
+ WhereInfo *pWInfo, /* The WHERE clause */
+ int iLevel, /* Index in pWInfo->a[] that is pLevel */
+ WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
){
- int addrTop;
- int addrCont;
- const WhereTerm *pTerm;
- const WhereTerm *pWCEnd;
- Parse *pParse = pWInfo->pParse;
- Vdbe *v = pParse->pVdbe;
- const WhereLoop *pLoop = pLevel->pWLoop;
- int iCur;
-
+ int addrOnce; /* Address of opening OP_Once */
+ int addrTop; /* Address of OP_Rewind */
+ int addrCont; /* Jump here to skip a row */
+ const WhereTerm *pTerm; /* For looping over WHERE clause terms */
+ const WhereTerm *pWCEnd; /* Last WHERE clause term */
+ Parse *pParse = pWInfo->pParse; /* Parsing context */
+ Vdbe *v = pParse->pVdbe; /* VDBE under construction */
+ WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
+ int iCur; /* Cursor for table getting the filter */
assert( pLoop!=0 );
assert( v!=0 );
- iCur = pLevel->iTabCur;
- addrCont = sqlite3VdbeMakeLabel(pParse);
- addrTop = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
- pLevel->regFilter = ++pParse->nMem;
- sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
- sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
- pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
- for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
- Expr *pExpr = pTerm->pExpr;
- if( (pTerm->wtFlags & TERM_VIRTUAL)==0
- && sqlite3ExprIsTableConstant(pExpr, iCur)
- ){
- sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
+ assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
+
+ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
+ do{
+ sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
+ addrCont = sqlite3VdbeMakeLabel(pParse);
+ iCur = pLevel->iTabCur;
+ pLevel->regFilter = ++pParse->nMem;
+ sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
+ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
+ pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
+ for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
+ Expr *pExpr = pTerm->pExpr;
+ if( (pTerm->wtFlags & TERM_VIRTUAL)==0
+ && sqlite3ExprIsTableConstant(pExpr, iCur)
+ ){
+ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
+ }
}
- }
- if( pLoop->wsFlags & WHERE_IPK ){
- int r1 = sqlite3GetTempReg(pParse);
- sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
- sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
- sqlite3ReleaseTempReg(pParse, r1);
- }else{
- Index *pIdx = pLoop->u.btree.pIndex;
- int n = pLoop->u.btree.nEq;
- int r1 = sqlite3GetTempRange(pParse, n);
- int jj;
- for(jj=0; jj<n; jj++){
- int iCol = pIdx->aiColumn[jj];
- sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
+ if( pLoop->wsFlags & WHERE_IPK ){
+ int r1 = sqlite3GetTempReg(pParse);
+ sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
+ sqlite3ReleaseTempReg(pParse, r1);
+ }else{
+ Index *pIdx = pLoop->u.btree.pIndex;
+ int n = pLoop->u.btree.nEq;
+ int r1 = sqlite3GetTempRange(pParse, n);
+ int jj;
+ for(jj=0; jj<n; jj++){
+ int iCol = pIdx->aiColumn[jj];
+ sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
+ }
+ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
+ sqlite3ReleaseTempRange(pParse, r1, n);
}
- sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
- sqlite3ReleaseTempRange(pParse, r1, n);
- }
- sqlite3VdbeResolveLabel(v, addrCont);
- sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
- VdbeCoverage(v);
- sqlite3VdbeJumpHere(v, addrTop);
- sqlite3VdbeJumpHere(v, addrTop+2);
+ sqlite3VdbeResolveLabel(v, addrCont);
+ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
+ VdbeCoverage(v);
+ sqlite3VdbeJumpHere(v, addrTop);
+ pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
+ if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
+ while( iLevel < pWInfo->nLevel ){
+ iLevel++;
+ pLevel = &pWInfo->a[iLevel];
+ pLoop = pLevel->pWLoop;
+ if( pLoop && pLoop->wsFlags & WHERE_BLOOMFILTER ) break;
+ }
+ }while( iLevel < pWInfo->nLevel );
+ sqlite3VdbeJumpHere(v, addrOnce);
}
@@ -5535,7 +5566,7 @@ WhereInfo *sqlite3WhereBegin(
&pTabList->a[pLevel->iFrom], notReady, pLevel);
#endif
}else{
- sqlite3ConstructBloomFilter(pWInfo, pLevel);
+ constructBloomFilter(pWInfo, ii, pLevel);
}
if( db->mallocFailed ) goto whereBeginError;
}
diff --git a/src/whereInt.h b/src/whereInt.h
index 19261ca397..4da6015f16 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -484,7 +484,6 @@ struct WhereInfo {
** where.c:
*/
Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
-void sqlite3ConstructBloomFilter(const WhereInfo*, WhereLevel*);
#ifdef WHERETRACE_ENABLED
void sqlite3WhereClausePrint(WhereClause *pWC);
void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm);
@@ -507,8 +506,14 @@ int sqlite3WhereExplainOneScan(
WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */
u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */
);
+int sqlite3WhereExplainBloomFilter(
+ const Parse *pParse, /* Parse context */
+ const WhereInfo *pWInfo, /* WHERE clause */
+ const WhereLevel *pLevel /* Bloom filter on this level */
+);
#else
# define sqlite3WhereExplainOneScan(u,v,w,x) 0
+# define sqlite3WhereExplainBloomFilter(u,v,w) 0
#endif /* SQLITE_OMIT_EXPLAIN */
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
void sqlite3WhereAddScanStatus(
diff --git a/src/wherecode.c b/src/wherecode.c
index adc7d5b8e1..dfe697f244 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -196,9 +196,6 @@ int sqlite3WhereExplainOneScan(
pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
}
#endif
- if( flags & WHERE_BLOOMFILTER ){
- sqlite3_str_appendf(&str, " WITH BLOOM FILTER");
- }
#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS
if( pLoop->nOut>=10 ){
sqlite3_str_appendf(&str, " (~%llu rows)",
@@ -214,6 +211,52 @@ int sqlite3WhereExplainOneScan(
}
return ret;
}
+
+/*
+** Add a single OP_Explain opcode that describes a Bloom filter.
+**
+** Or if not processing EXPLAIN QUERY PLAN and not in a SQLITE_DEBUG and/or
+** SQLITE_ENABLE_STMT_SCANSTATUS build, then OP_Explain opcodes are not
+** required and this routine is a no-op.
+**
+** If an OP_Explain opcode is added to the VM, its address is returned.
+** Otherwise, if no OP_Explain is coded, zero is returned.
+*/
+int sqlite3WhereExplainBloomFilter(
+ const Parse *pParse, /* Parse context */
+ const WhereInfo *pWInfo, /* WHERE clause */
+ const WhereLevel *pLevel /* Bloom filter on this level */
+){
+ int ret = 0;
+#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+ if( sqlite3ParseToplevel(pParse)->explain==2 )
+#endif
+ {
+ SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+ Vdbe *v = pParse->pVdbe; /* VM being constructed */
+ sqlite3 *db = pParse->db; /* Database handle */
+ char *zMsg; /* Text to add to EQP output */
+ int i; /* Loop counter */
+ WhereLoop *pLoop; /* The where loop */
+ StrAccum str; /* EQP output string */
+ char zBuf[100]; /* Initial space for EQP output string */
+
+ sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
+ str.printfFlags = SQLITE_PRINTF_INTERNAL;
+ sqlite3_str_appendf(&str, "BLOOM FILTER ON %S(", pItem);
+ pLoop = pLevel->pWLoop;
+ for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
+ const char *z = pItem->pTab->aCol[i].zCnName;
+ if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
+ sqlite3_str_appendf(&str, "%s=?", z);
+ }
+ sqlite3_str_append(&str, ")", 1);
+ zMsg = sqlite3StrAccumFinish(&str);
+ ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
+ pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
+ }
+ return ret;
+}
#endif /* SQLITE_OMIT_EXPLAIN */
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -1304,14 +1347,20 @@ static void whereApplyPartialIndexConstraints(
}
}
-#if 1
/*
-** An OP_Filter has just been generated, but the corresponding
-** index search has not yet been performed. This routine
-** checks to see if there are additional WHERE_BLOOMFILTER in
-** inner loops that can be evaluated right away, and if there are,
-** it evaluates those filters as well, and removes the WHERE_BLOOMFILTER
-** tag.
+** This routine is called right after An OP_Filter has been generated and
+** before the corresponding index search has been performed. This routine
+** checks to see if there are additional Bloom filters in inner loops that
+** can be checked prior to doing the index lookup. If there are available
+** inner-loop Bloom filters, then evaluate those filters now, before the
+** index lookup. The idea is that a Bloom filter check is way faster than
+** an index lookup, and the Bloom filter might return false, meaning that
+** the index lookup can be skipped.
+**
+** We know that an inner loop uses a Bloom filter because it has the
+** WhereLevel.regFilter set. If an inner-loop Bloom filter is checked,
+** then clear the WhereLoeve.regFilter value to prevent the Bloom filter
+** from being checked a second time when the inner loop is evaluated.
*/
static SQLITE_NOINLINE void filterPullDown(
Parse *pParse, /* Parsing context */
@@ -1323,9 +1372,8 @@ static SQLITE_NOINLINE void filterPullDown(
while( ++iLevel < pWInfo->nLevel ){
WhereLevel *pLevel = &pWInfo->a[iLevel];
WhereLoop *pLoop = pLevel->pWLoop;
- if( (pLoop->wsFlags & WHERE_BLOOMFILTER)==0 ) continue;
+ if( pLevel->regFilter==0 ) continue;
if( pLoop->prereq & notReady ) continue;
- sqlite3ConstructBloomFilter(pWInfo, &pWInfo->a[iLevel]);
if( pLoop->wsFlags & WHERE_IPK ){
WhereTerm *pTerm = pLoop->aLTerm[0];
int r1, regRowid;
@@ -1351,12 +1399,9 @@ static SQLITE_NOINLINE void filterPullDown(
addrNxt, r1, nEq);
VdbeCoverage(pParse->pVdbe);
}
- pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
+ pLevel->regFilter = 0;
}
}
-#else
-#define filterPullDown(A,B,C,D,E)
-#endif
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause
From 5baaf40af1898bdcfcdb5c30405b6da1342e2011 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 6 Dec 2021 13:07:28 +0000
Subject: [PATCH 09/18] Attempt to vary the size of Bloom filters based on an
estimate of how many keys the filter will hold.
FossilOrigin-Name: a7adcf69088cba4b86cc5731a45c9a5263af4355bc0a38f5225cab421c915f7f
---
manifest | 18 +++++------
manifest.uuid | 2 +-
src/malloc.c | 19 ++++++++++++
src/sqliteInt.h | 1 +
src/vdbe.c | 81 +++++++++++++++++++++++++++++++++++++------------
src/where.c | 5 +--
6 files changed, 95 insertions(+), 31 deletions(-)
diff --git a/manifest b/manifest
index 47992d2a2e..648c9a9161 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Run\sas\smany\sBloom\sfilters\sas\spossible\sbefore\sindex\slookups.
-D 2021-12-05T20:19:47.744
+C Attempt\sto\svary\sthe\ssize\sof\sBloom\sfilters\sbased\son\san\sestimate\sof\show\smany\nkeys\sthe\sfilter\swill\shold.
+D 2021-12-06T13:07:28.112
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -515,7 +515,7 @@ F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c
F src/main.c 7bd4fdc41ef53535271a1816ff043ba153cda03842b444b6e2f57b27b2cb9090
-F src/malloc.c ef796bcc0e81d845d59a469f1cf235056caf9024172fd524e32136e65593647b
+F src/malloc.c 183c2bf45cee1589254e4047e220f1ffbcc0a3bc8e4fe46fe64ba5db447a79af
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75
@@ -555,7 +555,7 @@ F src/shell.c.in e7ee6517544d075d9f06ee2571567026b89cf9fbeef16a74918019b1cb42576
F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h 178eb899c1edc08dcddf37e79dfaa39404a1f5d44a1d512509cd5d41867aa836
+F src/sqliteInt.h ab40ea9c294c656e0d6ab14e67d58f10b015a77e962dd075fdbe3ea3cc1a976b
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -622,7 +622,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c 94af4eba93ad9ca7dd929cd19792ce2a5feb4797a7a64ec3cb3b2277e1467a8b
+F src/vdbe.c 6176125ea038f593597b5897898328142b5253201d321369df74e187b2b1abaa
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
@@ -637,7 +637,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c b07c5eefecffa1b69b91c366a83c69d01a83f1c900b9d9b1ffb6eb5ab59902a1
+F src/where.c 04ead529a272341a4cae3ef0dcd2f7675d433627acc5fb87fed1407e7b3d8614
F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
@@ -1933,7 +1933,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5be2470f8755ef454f813c880e659bdbf82f2396be9320cf3079cd4ca8e81a19
-R da3cb867f9ab99abba060de93457c8d9
+P 06f6fefd67086896bc49272c6319545ff6c6792f18babe23aced27b60b032119
+R 5f90145148ed31800619d7b25d77beb7
U drh
-Z d87a509afa829d1cf21b5a6dcadef441
+Z 3013773c2d9a03770ac02d1fdd22abd3
diff --git a/manifest.uuid b/manifest.uuid
index 669a897f37..7894b82c00 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-06f6fefd67086896bc49272c6319545ff6c6792f18babe23aced27b60b032119
\ No newline at end of file
+a7adcf69088cba4b86cc5731a45c9a5263af4355bc0a38f5225cab421c915f7f
\ No newline at end of file
diff --git a/src/malloc.c b/src/malloc.c
index 932cecc210..ab9b37ddad 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -211,6 +211,25 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
return mx;
}
+/*
+** Return an estimate of the amount of unallocated memory.
+**
+** This the hard heap limit minus the current memory usage. It might
+** not be possible to allocate this much memory all at once. This is
+** only an estimate.
+*/
+sqlite3_int64 sqlite3EstMemoryAvailable(void){
+ sqlite3_int64 n;
+ sqlite3_mutex_enter(mem0.mutex);
+ n = mem0.alarmThreshold;
+ if( n<=0 ) n = mem0.hardLimit;
+ sqlite3_mutex_leave(mem0.mutex);
+ if( n<=0 ) n = LARGEST_INT64;
+ n -= sqlite3_memory_used();
+ if( n<0 ) n = 0;
+ return n;
+}
+
/*
** Trigger the alarm
*/
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 896b2aa422..9091f02115 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4293,6 +4293,7 @@ void sqlite3MemSetDefault(void);
void sqlite3BenignMallocHooks(void (*)(void), void (*)(void));
#endif
int sqlite3HeapNearlyFull(void);
+sqlite3_int64 sqlite3EstMemoryAvailable(void);
/*
** On systems with ample stack space and that support alloca(), make
diff --git a/src/vdbe.c b/src/vdbe.c
index acbbee892f..487d731af3 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -672,17 +672,29 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){
}
/*
-** Default size of a bloom filter, in bytes
+** The minimum size (in bytes) for a Bloom filter.
+**
+** No Bloom filter will be smaller than this many bytes. But they
+** may be larger.
*/
-#define SQLITE_BLOOM_SZ 10000
+#ifndef SQLITE_BLOOM_MIN
+# define SQLITE_BLOOM_MIN 10000
+#endif
+
+/*
+** The maximum size (in bytes) for a Bloom filter.
+*/
+#ifndef SQLITE_BLOOM_MAX
+# define SQLITE_BLOOM_MAX 1000000
+#endif
/*
** Compute a bloom filter hash using pOp->p4.i registers from aMem[] beginning
** with pOp->p3. Return the hash.
*/
-static unsigned int filterHash(const Mem *aMem, const Op *pOp){
+static u64 filterHash(const Mem *aMem, const Op *pOp){
int i, mx;
- u32 h = 0;
+ u64 h = 0;
i = pOp->p3;
assert( pOp->p4type==P4_INT32 );
@@ -690,15 +702,15 @@ static unsigned int filterHash(const Mem *aMem, const Op *pOp){
for(i=pOp->p3, mx=i+pOp->p4.i; i<mx; i++){
const Mem *p = &aMem[i];
if( p->flags & (MEM_Int|MEM_IntReal) ){
- h += (u32)(p->u.i&0xffffffff);
+ h += p->u.i;
}else if( p->flags & MEM_Real ){
- h += (u32)(sqlite3VdbeIntValue(p)&0xffffffff);
+ h += sqlite3VdbeIntValue(p);
}else if( p->flags & (MEM_Str|MEM_Blob) ){
h += p->n;
if( p->flags & MEM_Zero ) h += p->u.nZero;
}
}
- return h % (SQLITE_BLOOM_SZ*8);
+ return h;
}
/*
@@ -8157,15 +8169,44 @@ case OP_Function: { /* group */
break;
}
-/* Opcode: FilterInit P1 * * * *
-** Synopsis: filter(P1) = empty
+/* Opcode: FilterInit P1 P2 * * *
**
** Initialize register P1 so that is an empty bloom filter.
+**
+** If P2 is positive, it is a register that holds an estimate on
+** the number of entries to be added to the Bloom filter. The
+** Bloom filter is sized accordingly. If P2 is zero or negative,
+** then a default-size Bloom filter is created.
+**
+** It is ok for P1 and P2 to be the same register. In that case the
+** integer value originally in that register will be overwritten
+** with the new empty bloom filter.
*/
case OP_FilterInit: {
+ i64 n, mx;
assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
pIn1 = &aMem[pOp->p1];
- sqlite3VdbeMemSetZeroBlob(pIn1, SQLITE_BLOOM_SZ);
+ if( pOp->p2>0 ){
+ assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
+ n = sqlite3VdbeIntValue(&aMem[pOp->p2]);
+ if( n<SQLITE_BLOOM_MIN ){
+ n = SQLITE_BLOOM_MIN;
+ }else if( n>SQLITE_BLOOM_MAX ){
+ n = SQLITE_BLOOM_MAX;
+ }
+ }else{
+ n = SQLITE_BLOOM_MIN;
+ }
+ mx = sqlite3EstMemoryAvailable()/2;
+ if( n>mx && mx>SQLITE_BLOOM_MIN ){
+ n = mx;
+ }
+#ifdef SQLITE_DEBUG
+ if( db->flags&SQLITE_VdbeTrace ){
+ printf("Bloom-filter size: %llu bytes\n", n);
+ }
+#endif
+ sqlite3VdbeMemSetZeroBlob(pIn1, n);
if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem;
break;
}
@@ -8177,12 +8218,12 @@ case OP_FilterInit: {
** add that hash to the bloom filter contained in r[P1].
*/
case OP_FilterAdd: {
- u32 h;
+ u64 h;
assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
pIn1 = &aMem[pOp->p1];
assert( pIn1->flags & MEM_Blob );
- assert( pIn1->n==SQLITE_BLOOM_SZ );
+ assert( pIn1->n>0 );
h = filterHash(aMem, pOp);
#ifdef SQLITE_DEBUG
if( db->flags&SQLITE_VdbeTrace ){
@@ -8190,10 +8231,10 @@ case OP_FilterAdd: {
for(ii=pOp->p3; ii<pOp->p3+pOp->p4.i; ii++){
registerTrace(ii, &aMem[ii]);
}
- printf("hash = %u\n", h);
+ printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n));
}
#endif
- assert( h>=0 && h<SQLITE_BLOOM_SZ*8 );
+ h %= pIn1->n;
pIn1->z[h/8] |= 1<<(h&7);
break;
}
@@ -8213,12 +8254,14 @@ case OP_FilterAdd: {
** false positive - if the jump is taken when it should fall through.
*/
case OP_Filter: { /* jump */
- u32 h;
+ u64 h;
assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
pIn1 = &aMem[pOp->p1];
- assert( pIn1->flags & MEM_Blob );
- assert( pIn1->n==SQLITE_BLOOM_SZ );
+ if( (pIn1->flags & MEM_Blob)==0 || NEVER(pIn1->n<=0) ){
+ VdbeBranchTaken(0, 2);
+ break;
+ }
h = filterHash(aMem, pOp);
#ifdef SQLITE_DEBUG
if( db->flags&SQLITE_VdbeTrace ){
@@ -8226,10 +8269,10 @@ case OP_Filter: { /* jump */
for(ii=pOp->p3; ii<pOp->p3+pOp->p4.i; ii++){
registerTrace(ii, &aMem[ii]);
}
- printf("hash = %u\n", h);
+ printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n));
}
#endif
- assert( h>=0 && h<SQLITE_BLOOM_SZ*8 );
+ h %= pIn1->n;
if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){
VdbeBranchTaken(1, 2);
goto jump_to_p2;
diff --git a/src/where.c b/src/where.c
index 1d88c3ed30..233e77b6a7 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1010,8 +1010,9 @@ static SQLITE_NOINLINE void constructBloomFilter(
addrCont = sqlite3VdbeMakeLabel(pParse);
iCur = pLevel->iTabCur;
pLevel->regFilter = ++pParse->nMem;
- sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
+ sqlite3VdbeAddOp3(v, OP_Count, iCur, pLevel->regFilter, 1);
+ sqlite3VdbeAddOp2(v, OP_FilterInit, pLevel->regFilter, pLevel->regFilter);
pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
Expr *pExpr = pTerm->pExpr;
@@ -1039,7 +1040,7 @@ static SQLITE_NOINLINE void constructBloomFilter(
sqlite3ReleaseTempRange(pParse, r1, n);
}
sqlite3VdbeResolveLabel(v, addrCont);
- sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
+ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
VdbeCoverage(v);
sqlite3VdbeJumpHere(v, addrTop);
pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
From 50fb7e09b4247ce34a0b0266614303d00f929def Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 6 Dec 2021 20:16:53 +0000
Subject: [PATCH 10/18] Omit the OP_FilterInit opcode. Use OP_Blob to
initialize each Bloom filter instead. Size the Bloom filter based on
sqlite_stat1 size estimates rather than a run-time measurement for improved
testability.
FossilOrigin-Name: 8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97
---
manifest | 20 +++++++++---------
manifest.uuid | 2 +-
src/malloc.c | 2 ++
src/sqliteInt.h | 7 +++----
src/util.c | 14 -------------
src/vdbe.c | 54 +++++++++----------------------------------------
src/where.c | 35 ++++++++++++++++++++++++++++----
7 files changed, 56 insertions(+), 78 deletions(-)
diff --git a/manifest b/manifest
index 754a38701e..b4f5f5334e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\sfixes\sinto\sthe\sbloom-filter\sbranch.
-D 2021-12-06T19:11:31.036
+C Omit\sthe\sOP_FilterInit\sopcode.\s\sUse\sOP_Blob\sto\sinitialize\seach\sBloom\sfilter\ninstead.\s\sSize\sthe\sBloom\sfilter\sbased\son\ssqlite_stat1\ssize\sestimates\srather\nthan\sa\srun-time\smeasurement\sfor\simproved\stestability.
+D 2021-12-06T20:16:53.814
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -516,7 +516,7 @@ F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c
F src/main.c 1ea70751e6005ab6a9f784730fa0919efaa6639440a287deb73cb711e5aae57a
-F src/malloc.c 183c2bf45cee1589254e4047e220f1ffbcc0a3bc8e4fe46fe64ba5db447a79af
+F src/malloc.c d9172a3946f11384f2fd6a799554ee26c6bb407c4bd0874a456ed485a2e362e4
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75
@@ -556,7 +556,7 @@ F src/shell.c.in 1458b700144c8326fda2514aaddeda49d6f01f1d1ccf7b9b696c53a3535a119
F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h ab40ea9c294c656e0d6ab14e67d58f10b015a77e962dd075fdbe3ea3cc1a976b
+F src/sqliteInt.h f4fbb14ea32d57b813aabf82f586d2ac042234dd89df1c03281f557907745b98
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -621,9 +621,9 @@ F src/trigger.c 2ef56f0b7b75349a5557d0604b475126329c2e1a02432e7d49c4c710613e8254
F src/update.c d6f5c7b9e072660757ac7d58175aca11c07cb95ebbb297ae7f38853700f52328
F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167
+F src/util.c 6dfbd0bd1954e9531e1c511e5d20390d7dab9ffbf1e20a37c960d1aaf8582b46
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c 6176125ea038f593597b5897898328142b5253201d321369df74e187b2b1abaa
+F src/vdbe.c 9cc221ebae54417b0b47b4ce6aa6eba7e919153f8ae6307f8d18a6749d453f9c
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
@@ -638,7 +638,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 04ead529a272341a4cae3ef0dcd2f7675d433627acc5fb87fed1407e7b3d8614
+F src/where.c 21bd1078837afb127827243d7ad549a4b47022ffaa43c5baa74dcac7f89809a7
F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7adcf69088cba4b86cc5731a45c9a5263af4355bc0a38f5225cab421c915f7f bb9b1a15f7e80483162049dfd981d059dc69d03348b521f7ac164a8cd3ae3cc4
-R 0f448dc1f18098ffcb6b2fdc27d44d39
+P edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed
+R 933b551d7a60e6c29e7de8b534d33928
U drh
-Z a01a7e207ad8c3ff4f1aefb648f1cc10
+Z ded7cd3093221f3f9f8f684eb1a7389d
diff --git a/manifest.uuid b/manifest.uuid
index 63d6d10d76..0b0fb52902 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed
\ No newline at end of file
+8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97
\ No newline at end of file
diff --git a/src/malloc.c b/src/malloc.c
index ab9b37ddad..9a2eaeec15 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -211,6 +211,7 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
return mx;
}
+#if 0
/*
** Return an estimate of the amount of unallocated memory.
**
@@ -229,6 +230,7 @@ sqlite3_int64 sqlite3EstMemoryAvailable(void){
if( n<0 ) n = 0;
return n;
}
+#endif
/*
** Trigger the alarm
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 9091f02115..90e60d30b0 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3927,6 +3927,7 @@ struct Sqlite3Config {
int iOnceResetThreshold; /* When to reset OP_Once counters */
u32 szSorterRef; /* Min size in bytes to use sorter-refs */
unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */
+ int iEstCountScale; /* Multiple RowCountEst() by this amount */
/* vvvv--- must be last ---vvv */
#ifdef SQLITE_DEBUG
sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */
@@ -4293,7 +4294,9 @@ void sqlite3MemSetDefault(void);
void sqlite3BenignMallocHooks(void (*)(void), void (*)(void));
#endif
int sqlite3HeapNearlyFull(void);
+#if 0
sqlite3_int64 sqlite3EstMemoryAvailable(void);
+#endif
/*
** On systems with ample stack space and that support alloca(), make
@@ -4780,11 +4783,7 @@ LogEst sqlite3LogEstAdd(LogEst,LogEst);
#ifndef SQLITE_OMIT_VIRTUALTABLE
LogEst sqlite3LogEstFromDouble(double);
#endif
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
- defined(SQLITE_ENABLE_STAT4) || \
- defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
u64 sqlite3LogEstToInt(LogEst);
-#endif
VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int);
const char *sqlite3VListNumToName(VList*,int);
int sqlite3VListNameToNum(VList*,const char*,int);
diff --git a/src/util.c b/src/util.c
index 8452aea665..d93c298116 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1603,14 +1603,8 @@ LogEst sqlite3LogEstFromDouble(double x){
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
- defined(SQLITE_ENABLE_STAT4) || \
- defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
/*
** Convert a LogEst into an integer.
-**
-** Note that this routine is only used when one or more of various
-** non-standard compile-time options is enabled.
*/
u64 sqlite3LogEstToInt(LogEst x){
u64 n;
@@ -1618,17 +1612,9 @@ u64 sqlite3LogEstToInt(LogEst x){
x /= 10;
if( n>=5 ) n -= 2;
else if( n>=1 ) n -= 1;
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
- defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
if( x>60 ) return (u64)LARGEST_INT64;
-#else
- /* If only SQLITE_ENABLE_STAT4 is on, then the largest input
- ** possible to this routine is 310, resulting in a maximum x of 31 */
- assert( x<=60 );
-#endif
return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x);
}
-#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */
/*
** Add a new name/number pair to a VList. This might require that the
diff --git a/src/vdbe.c b/src/vdbe.c
index 487d731af3..c2281e70a1 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -1367,12 +1367,18 @@ case OP_SoftNull: {
** Synopsis: r[P2]=P4 (len=P1)
**
** P4 points to a blob of data P1 bytes long. Store this
-** blob in register P2.
+** blob in register P2. If P4 is a NULL pointer, then construct
+** a zero-filled blob that is P1 bytes long in P2.
*/
case OP_Blob: { /* out2 */
assert( pOp->p1 <= SQLITE_MAX_LENGTH );
pOut = out2Prerelease(p, pOp);
- sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0);
+ if( pOp->p4.z==0 ){
+ sqlite3VdbeMemSetZeroBlob(pOut, pOp->p1);
+ if( sqlite3VdbeMemExpandBlob(pOut) ) goto no_mem;
+ }else{
+ sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0);
+ }
pOut->enc = encoding;
UPDATE_MAX_BLOBSIZE(pOut);
break;
@@ -3383,7 +3389,7 @@ case OP_MakeRecord: {
break;
}
-/* Opcode: Count P1 P2 p3 * *
+/* Opcode: Count P1 P2 P3 * *
** Synopsis: r[P2]=count()
**
** Store the number of entries (an integer value) in the table or index
@@ -8169,48 +8175,6 @@ case OP_Function: { /* group */
break;
}
-/* Opcode: FilterInit P1 P2 * * *
-**
-** Initialize register P1 so that is an empty bloom filter.
-**
-** If P2 is positive, it is a register that holds an estimate on
-** the number of entries to be added to the Bloom filter. The
-** Bloom filter is sized accordingly. If P2 is zero or negative,
-** then a default-size Bloom filter is created.
-**
-** It is ok for P1 and P2 to be the same register. In that case the
-** integer value originally in that register will be overwritten
-** with the new empty bloom filter.
-*/
-case OP_FilterInit: {
- i64 n, mx;
- assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
- pIn1 = &aMem[pOp->p1];
- if( pOp->p2>0 ){
- assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
- n = sqlite3VdbeIntValue(&aMem[pOp->p2]);
- if( n<SQLITE_BLOOM_MIN ){
- n = SQLITE_BLOOM_MIN;
- }else if( n>SQLITE_BLOOM_MAX ){
- n = SQLITE_BLOOM_MAX;
- }
- }else{
- n = SQLITE_BLOOM_MIN;
- }
- mx = sqlite3EstMemoryAvailable()/2;
- if( n>mx && mx>SQLITE_BLOOM_MIN ){
- n = mx;
- }
-#ifdef SQLITE_DEBUG
- if( db->flags&SQLITE_VdbeTrace ){
- printf("Bloom-filter size: %llu bytes\n", n);
- }
-#endif
- sqlite3VdbeMemSetZeroBlob(pIn1, n);
- if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem;
- break;
-}
-
/* Opcode: FilterAdd P1 * P3 P4 *
** Synopsis: filter(P1) += key(P3@P4)
**
diff --git a/src/where.c b/src/where.c
index 233e77b6a7..e7246b1040 100644
--- a/src/where.c
+++ b/src/where.c
@@ -911,7 +911,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
VdbeComment((v, "for %s", pTable->zName));
if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){
pLevel->regFilter = ++pParse->nMem;
- sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
+ sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
}
/* Fill the automatic index with content */
@@ -1006,13 +1006,39 @@ static SQLITE_NOINLINE void constructBloomFilter(
addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
do{
+ const SrcItem *pItem;
+ const Table *pTab;
+ int sz;
sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
addrCont = sqlite3VdbeMakeLabel(pParse);
iCur = pLevel->iTabCur;
pLevel->regFilter = ++pParse->nMem;
+
+ /* The Bloom filter is a Blob held in a register. Initialize it
+ ** to zero-filled blob of at least 80K bits, but maybe more if the
+ ** estimated size of the table is larger. We could actually
+ ** measure the size of the table at run-time using OP_Count with
+ ** P3==1 and use that value to initialize the blob. But that makes
+ ** testing complicated. By basing the blob size on the value in the
+ ** sqlite_stat1 table, testing is much easier.
+ */
+ pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+ assert( pItem!=0 );
+ pTab = pItem->pTab;
+ assert( pTab!=0 );
+ if( pTab->tabFlags & TF_HasStat1 ){
+ sz = sqlite3LogEstToInt(pItem->pTab->nRowLogEst);
+ if( sz<10000 ){
+ sz = 10000;
+ }else if( sz>10000000 ){
+ sz = 10000000;
+ }
+ }else{
+ sz = 10000;
+ }
+ sqlite3VdbeAddOp2(v, OP_Blob, sz, pLevel->regFilter);
+
addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
- sqlite3VdbeAddOp3(v, OP_Count, iCur, pLevel->regFilter, 1);
- sqlite3VdbeAddOp2(v, OP_FilterInit, pLevel->regFilter, pLevel->regFilter);
pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
Expr *pExpr = pTerm->pExpr;
@@ -1034,13 +1060,14 @@ static SQLITE_NOINLINE void constructBloomFilter(
int jj;
for(jj=0; jj<n; jj++){
int iCol = pIdx->aiColumn[jj];
+ assert( pIdx->pTable==pItem->pTab );
sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
}
sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
sqlite3ReleaseTempRange(pParse, r1, n);
}
sqlite3VdbeResolveLabel(v, addrCont);
- sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
+ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
VdbeCoverage(v);
sqlite3VdbeJumpHere(v, addrTop);
pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
From 23d41e63f2e4560728e13cefabb4fd411c687672 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 6 Dec 2021 21:45:31 +0000
Subject: [PATCH 11/18] Add SQLITE_STMTSTATUS_FILTER_HIT and _MISS for tracking
the effectiveness of Bloom filters.
FossilOrigin-Name: 24ba535d200fc8a99dd8e66c6d100b5f6ae442098bafb152008429398eefe3e7
---
manifest | 18 +++++++++---------
manifest.uuid | 2 +-
src/shell.c.in | 7 +++++++
src/sqlite.h.in | 9 +++++++++
src/vdbe.c | 2 ++
src/vdbeInt.h | 2 +-
6 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/manifest b/manifest
index b4f5f5334e..956bcffef9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Omit\sthe\sOP_FilterInit\sopcode.\s\sUse\sOP_Blob\sto\sinitialize\seach\sBloom\sfilter\ninstead.\s\sSize\sthe\sBloom\sfilter\sbased\son\ssqlite_stat1\ssize\sestimates\srather\nthan\sa\srun-time\smeasurement\sfor\simproved\stestability.
-D 2021-12-06T20:16:53.814
+C Add\sSQLITE_STMTSTATUS_FILTER_HIT\sand\s_MISS\sfor\stracking\sthe\seffectiveness\nof\sBloom\sfilters.
+D 2021-12-06T21:45:31.659
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -552,8 +552,8 @@ F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c
F src/resolve.c 4a1db4aadd802683db40ca2dbbb268187bd195f10cbdb7206dbd8ac988795571
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c a7a3d9f54eb24821ec5f67f2e5589b68a5d42d46fc5849d7376886777d93a85a
-F src/shell.c.in 1458b700144c8326fda2514aaddeda49d6f01f1d1ccf7b9b696c53a3535a119c
-F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
+F src/shell.c.in cda1eaf0292259b4b0721a5e03af9701fd482ebc37ce6a86ddc94cd9a38bb826
+F src/sqlite.h.in bb56040e3c498711c9f77727e477674395a50931ccba8095cfef5c8fb3c3e138
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
F src/sqliteInt.h f4fbb14ea32d57b813aabf82f586d2ac042234dd89df1c03281f557907745b98
@@ -623,9 +623,9 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
F src/util.c 6dfbd0bd1954e9531e1c511e5d20390d7dab9ffbf1e20a37c960d1aaf8582b46
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c 9cc221ebae54417b0b47b4ce6aa6eba7e919153f8ae6307f8d18a6749d453f9c
+F src/vdbe.c fd8542b7131f299659871535a41ea732764fb25e4d2931965c97fa36658c50d7
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
-F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb
+F src/vdbeInt.h 910985ac2783fe0938b314d811759d53fd25caf215810f62ca1ff068d6d60d7b
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
F src/vdbeaux.c 21db442d159fd745a7693d157b5f998260b6af4ca60de559fa3b7b68c7405af2
F src/vdbeblob.c 29c4118f7ee615cdee829e8401f6ead1b96b95d545b4de0042f6de39c962c652
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed
-R 933b551d7a60e6c29e7de8b534d33928
+P 8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97
+R ec73578237c4366b7d0d282bfae04830
U drh
-Z ded7cd3093221f3f9f8f684eb1a7389d
+Z e55f2c8840b9a35831f295372358b466
diff --git a/manifest.uuid b/manifest.uuid
index 0b0fb52902..71816720c8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97
\ No newline at end of file
+24ba535d200fc8a99dd8e66c6d100b5f6ae442098bafb152008429398eefe3e7
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 543141c9e4..c46eaffd62 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2790,6 +2790,7 @@ static int display_stats(
}
if( pArg->pStmt ){
+ int iHit, iMiss;
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP,
bReset);
raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur);
@@ -2797,6 +2798,12 @@ static int display_stats(
raw_printf(pArg->out, "Sort Operations: %d\n", iCur);
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset);
raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur);
+ iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset);
+ iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset);
+ if( iHit || iMiss ){
+ raw_printf(pArg->out, "Bloom filter bypass taken: %d/%d\n",
+ iHit, iHit+iMiss);
+ }
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset);
raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur);
iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset);
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 98a028b0b0..4125122e17 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8467,6 +8467,13 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
** The counter is incremented on the first [sqlite3_step()] call of each
** cycle.
**
+** [[SQLITE_STMTSTATUS_FILTER HIT]] <dt>SQLITE_STMTSTATUS_FILTER_HIT</dt>
+** <dd>^This is the number of times that a join step was bypassed because
+** a Bloom filtered returned non-found. The corresponding
+** SQLITE_STMTSTATUS_FILTER_MISS value is the number of times that the
+** Bloom filter returned a find, and thus the join step had to be processed
+** as normal.
+**
** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt>
** <dd>^This is the approximate number of bytes of heap memory
** used to store the prepared statement. ^This value is not actually
@@ -8481,6 +8488,8 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
#define SQLITE_STMTSTATUS_VM_STEP 4
#define SQLITE_STMTSTATUS_REPREPARE 5
#define SQLITE_STMTSTATUS_RUN 6
+#define SQLITE_STMTSTATUS_FILTER_MISS 7
+#define SQLITE_STMTSTATUS_FILTER_HIT 8
#define SQLITE_STMTSTATUS_MEMUSED 99
/*
diff --git a/src/vdbe.c b/src/vdbe.c
index c2281e70a1..0abe64a71f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -8239,8 +8239,10 @@ case OP_Filter: { /* jump */
h %= pIn1->n;
if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){
VdbeBranchTaken(1, 2);
+ p->aCounter[SQLITE_STMTSTATUS_FILTER_HIT]++;
goto jump_to_p2;
}else{
+ p->aCounter[SQLITE_STMTSTATUS_FILTER_MISS]++;
VdbeBranchTaken(0, 2);
}
break;
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index c76cdbfdbc..38863f6d65 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -430,7 +430,7 @@ struct Vdbe {
bft bIsReader:1; /* True for statements that read */
yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */
yDbMask lockMask; /* Subset of btreeMask that requires a lock */
- u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */
+ u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */
char *zSql; /* Text of the SQL statement that generated this */
#ifdef SQLITE_ENABLE_NORMALIZE
char *zNormSql; /* Normalization of the associated SQL statement */
From 3bd7cd736d142ec3068f56cac510afdbc85c925d Mon Sep 17 00:00:00 2001
From: drh <>
Date: Mon, 6 Dec 2021 23:07:59 +0000
Subject: [PATCH 12/18] Improved EXPLAIN QUERY PLAN output for Bloom filters.
FossilOrigin-Name: 00070e1fff6aec3d7c7b121f2b02bbca38a1664aca9afc3fb7e293f07fd1704f
---
manifest | 12 ++++++------
manifest.uuid | 2 +-
src/wherecode.c | 41 +++++++++++++++++++++++++++++------------
3 files changed, 36 insertions(+), 19 deletions(-)
diff --git a/manifest b/manifest
index 956bcffef9..6a4298ce44 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sSQLITE_STMTSTATUS_FILTER_HIT\sand\s_MISS\sfor\stracking\sthe\seffectiveness\nof\sBloom\sfilters.
-D 2021-12-06T21:45:31.659
+C Improved\sEXPLAIN\sQUERY\sPLAN\soutput\sfor\sBloom\sfilters.
+D 2021-12-06T23:07:59.482
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -640,7 +640,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
F src/where.c 21bd1078837afb127827243d7ad549a4b47022ffaa43c5baa74dcac7f89809a7
F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
-F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c
+F src/wherecode.c f82a322a8849a0290587d968f5c1c71b0d5018e078f43ea732a4cdbf837ed42b
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97
-R ec73578237c4366b7d0d282bfae04830
+P 24ba535d200fc8a99dd8e66c6d100b5f6ae442098bafb152008429398eefe3e7
+R 7eae2a1d2a9d2b173d70c0467bdac4ba
U drh
-Z e55f2c8840b9a35831f295372358b466
+Z 477e7fdac36f7f761dc02a6457a5f233
diff --git a/manifest.uuid b/manifest.uuid
index 71816720c8..a54777d56d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-24ba535d200fc8a99dd8e66c6d100b5f6ae442098bafb152008429398eefe3e7
\ No newline at end of file
+00070e1fff6aec3d7c7b121f2b02bbca38a1664aca9afc3fb7e293f07fd1704f
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index dfe697f244..c04a73c2a8 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -176,19 +176,27 @@ int sqlite3WhereExplainOneScan(
explainIndexRange(&str, pLoop);
}
}else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){
- const char *zRangeOp;
+ char cRangeOp;
+#if 0 /* Better output, but breaks many tests */
+ const Table *pTab = pItem->pTab;
+ const char *zRowid = pTab->iPKey>=0 ? pTab->aCol[pTab->iPKey].zCnName:
+ "rowid";
+#else
+ const char *zRowid = "rowid";
+#endif
+ sqlite3_str_appendf(&str, " USING INTEGER PRIMARY KEY (%s", zRowid);
if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){
- zRangeOp = "=";
+ cRangeOp = '=';
}else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
- zRangeOp = ">? AND rowid<";
+ sqlite3_str_appendf(&str, ">? AND %s", zRowid);
+ cRangeOp = '<';
}else if( flags&WHERE_BTM_LIMIT ){
- zRangeOp = ">";
+ cRangeOp = '>';
}else{
assert( flags&WHERE_TOP_LIMIT);
- zRangeOp = "<";
+ cRangeOp = '<';
}
- sqlite3_str_appendf(&str,
- " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp);
+ sqlite3_str_appendf(&str, "%c?)", cRangeOp);
}
#ifndef SQLITE_OMIT_VIRTUALTABLE
else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
@@ -243,12 +251,21 @@ int sqlite3WhereExplainBloomFilter(
sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
str.printfFlags = SQLITE_PRINTF_INTERNAL;
- sqlite3_str_appendf(&str, "BLOOM FILTER ON %S(", pItem);
+ sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem);
pLoop = pLevel->pWLoop;
- for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
- const char *z = pItem->pTab->aCol[i].zCnName;
- if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
- sqlite3_str_appendf(&str, "%s=?", z);
+ if( pLoop->wsFlags & WHERE_IPK ){
+ const Table *pTab = pItem->pTab;
+ if( pTab->iPKey>=0 ){
+ sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName);
+ }else{
+ sqlite3_str_appendf(&str, "rowid=?");
+ }
+ }else{
+ for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
+ const char *z = explainIndexColumnName(pLoop->u.btree.pIndex, i);
+ if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
+ sqlite3_str_appendf(&str, "%s=?", z);
+ }
}
sqlite3_str_append(&str, ")", 1);
zMsg = sqlite3StrAccumFinish(&str);
From 761d64b73bc7ceb5ef60d956b6e1b78c1997082d Mon Sep 17 00:00:00 2001
From: drh <>
Date: Tue, 7 Dec 2021 22:37:50 +0000
Subject: [PATCH 13/18] Do not generate a Bloom filter if it cannot be used
prior to the next seek, as that leads to a misleading EXPLAIN QUERY PLAN.
FossilOrigin-Name: 2739ed5192058fbcc816ecbc252be687efc606e038bfcd6cf71194a3f4f5684e
---
manifest | 12 ++++++------
manifest.uuid | 2 +-
src/where.c | 9 ++++++---
3 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/manifest b/manifest
index 6a4298ce44..78850cff06 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sEXPLAIN\sQUERY\sPLAN\soutput\sfor\sBloom\sfilters.
-D 2021-12-06T23:07:59.482
+C Do\snot\sgenerate\sa\sBloom\sfilter\sif\sit\scannot\sbe\sused\sprior\sto\sthe\snext\sseek,\nas\sthat\sleads\sto\sa\smisleading\sEXPLAIN\sQUERY\sPLAN.
+D 2021-12-07T22:37:50.686
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -638,7 +638,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 21bd1078837afb127827243d7ad549a4b47022ffaa43c5baa74dcac7f89809a7
+F src/where.c 328e5c6f5a91f8c1e36417159cc93ea180700ae97726f934f69e68c8c60030b5
F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
F src/wherecode.c f82a322a8849a0290587d968f5c1c71b0d5018e078f43ea732a4cdbf837ed42b
F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 24ba535d200fc8a99dd8e66c6d100b5f6ae442098bafb152008429398eefe3e7
-R 7eae2a1d2a9d2b173d70c0467bdac4ba
+P 00070e1fff6aec3d7c7b121f2b02bbca38a1664aca9afc3fb7e293f07fd1704f
+R 6ef1a6c68af918fb5dc855e869b3769c
U drh
-Z 477e7fdac36f7f761dc02a6457a5f233
+Z 7a26146d090468562610338a13462117
diff --git a/manifest.uuid b/manifest.uuid
index a54777d56d..f1d3a5f29a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-00070e1fff6aec3d7c7b121f2b02bbca38a1664aca9afc3fb7e293f07fd1704f
\ No newline at end of file
+2739ed5192058fbcc816ecbc252be687efc606e038bfcd6cf71194a3f4f5684e
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index e7246b1040..1eb9f36017 100644
--- a/src/where.c
+++ b/src/where.c
@@ -988,7 +988,8 @@ end_auto_index_create:
static SQLITE_NOINLINE void constructBloomFilter(
WhereInfo *pWInfo, /* The WHERE clause */
int iLevel, /* Index in pWInfo->a[] that is pLevel */
- WhereLevel *pLevel /* Make a Bloom filter for this FROM term */
+ WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
+ Bitmask notReady /* Loops that are not ready */
){
int addrOnce; /* Address of opening OP_Once */
int addrTop; /* Address of OP_Rewind */
@@ -1076,7 +1077,9 @@ static SQLITE_NOINLINE void constructBloomFilter(
iLevel++;
pLevel = &pWInfo->a[iLevel];
pLoop = pLevel->pWLoop;
- if( pLoop && pLoop->wsFlags & WHERE_BLOOMFILTER ) break;
+ if( pLoop==0 ) continue;
+ if( pLoop->prereq & notReady ) continue;
+ if( pLoop->wsFlags & WHERE_BLOOMFILTER ) break;
}
}while( iLevel < pWInfo->nLevel );
sqlite3VdbeJumpHere(v, addrOnce);
@@ -5594,7 +5597,7 @@ WhereInfo *sqlite3WhereBegin(
&pTabList->a[pLevel->iFrom], notReady, pLevel);
#endif
}else{
- constructBloomFilter(pWInfo, ii, pLevel);
+ constructBloomFilter(pWInfo, ii, pLevel, notReady);
}
if( db->mallocFailed ) goto whereBeginError;
}
From fb82caf0bed2cd006cda440888939b87641b00c9 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Wed, 8 Dec 2021 19:50:45 +0000
Subject: [PATCH 14/18] Improvements on the decision of whether or not to use a
Bloom filter.
FossilOrigin-Name: 0fb2a4e08f518cb38ea3edc6a084d1e4874fd622ba3cf9101b49b3e7dc1a3f2b
---
manifest | 14 +++++++-------
manifest.uuid | 2 +-
src/where.c | 35 ++++++++++++++++++++++++-----------
src/whereInt.h | 1 +
4 files changed, 33 insertions(+), 19 deletions(-)
diff --git a/manifest b/manifest
index ed505a6910..4c4a9d684d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\sWhereClause\sfix\sfrom\strunk
-D 2021-12-08T16:15:41.698
+C Improvements\son\sthe\sdecision\sof\swhether\sor\snot\sto\suse\sa\sBloom\sfilter.
+D 2021-12-08T19:50:45.145
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -638,8 +638,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 74382d720f2a9a06f738467929ecf0d79fcdd6b9779184bddbb97352895a1b23
-F src/whereInt.h d55d5ce5c9de361e16cf5cd23da054cdecc1ae7735682ee55c2f40e58e960d22
+F src/where.c 89958d4fc7c45e916882ebc97481d98597f516ce3d778ace3271aacf34e24e91
+F src/whereInt.h c2cb535e755b25a7e152bdb407cbb2f62bdb8747c44bf2d984139f5cbebb8704
F src/wherecode.c e2207f011b7e5bdef5722da5e8d95eb30ad01051b3526757901ecb19a9e98ff3
F src/whereexpr.c 791544603b254cf11f8e84e3b50b0863c57322e9f213b828680f658e232ebc57
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2739ed5192058fbcc816ecbc252be687efc606e038bfcd6cf71194a3f4f5684e 6024682ca467fa4fe49608772b0bbfa2f8a419b32cebfa715941073c8b29da49
-R d3afb24f2c63fc5d59ade7856914b3ad
+P d3250256772e3348abe887c0ca3550a6647cce3804c9456a9d0112aea7ee1c46
+R a7fca9e9fc2528f3e921fdfbeaa2e361
U drh
-Z 246d691185204c115a5a684ad14acaf7
+Z 815ca39cf2b788796cb5ebc9be23c44d
diff --git a/manifest.uuid b/manifest.uuid
index 1416a80b0a..a9d9e689b3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d3250256772e3348abe887c0ca3550a6647cce3804c9456a9d0112aea7ee1c46
\ No newline at end of file
+0fb2a4e08f518cb38ea3edc6a084d1e4874fd622ba3cf9101b49b3e7dc1a3f2b
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index bf10262a76..065d14fd64 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2028,9 +2028,9 @@ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
sqlite3_free(z);
}
if( p->wsFlags & WHERE_SKIPSCAN ){
- sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
+ sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
}else{
- sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm);
+ sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
}
sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){
@@ -2502,6 +2502,9 @@ static void whereLoopOutputAdjust(
if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
}
if( j<0 ){
+ if( pLoop->maskSelf==pTerm->prereqAll ){
+ pLoop->wsFlags |= WHERE_CULLED;
+ }
if( pTerm->truthProb<=0 ){
/* If a truth probability is specified using the likelihood() hints,
** then use the probability provided by the application. */
@@ -2529,7 +2532,9 @@ static void whereLoopOutputAdjust(
}
}
}
- if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce;
+ if( pLoop->nOut > nRow-iReduce ){
+ pLoop->nOut = nRow - iReduce;
+ }
}
/*
@@ -4984,22 +4989,30 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
){
int i;
LogEst nSearch;
- SrcItem *pItem;
assert( pWInfo->nLevel>=2 );
assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
nSearch = pWInfo->a[0].pWLoop->nOut;
for(i=1; i<pWInfo->nLevel; i++){
WhereLoop *pLoop = pWInfo->a[i].pWLoop;
- if( pLoop->nOut<0
+ const int reqFlags = (WHERE_CULLED|WHERE_COLUMN_EQ);
+ if( (pLoop->wsFlags & reqFlags)==reqFlags
&& (pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0
- && (pLoop->wsFlags & WHERE_COLUMN_EQ)!=0
- && nSearch > (pItem = &pWInfo->pTabList->a[pLoop->iTab])->pTab->nRowLogEst
- && (pItem->fg.jointype & JT_LEFT)==0
){
- pLoop->wsFlags |= WHERE_BLOOMFILTER;
- pLoop->wsFlags &= ~WHERE_IDX_ONLY;
- WHERETRACE(0xffff, ("-> use Bloom-filter on loop %c\n", pLoop->cId));
+ SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
+ Table *pTab = pItem->pTab;
+ pTab->tabFlags |= TF_StatsUsed;
+ if( nSearch > pTab->nRowLogEst
+ && (pItem->fg.jointype & JT_LEFT)==0
+ ){
+ pLoop->wsFlags |= WHERE_BLOOMFILTER;
+ pLoop->wsFlags &= ~WHERE_IDX_ONLY;
+ WHERETRACE(0xffff, (
+ "-> use Bloom-filter on loop %c because there are ~%.1e "
+ "lookups into %s which has only ~%.1e rows\n",
+ pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
+ (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
+ }
}
nSearch += pLoop->nOut;
}
diff --git a/src/whereInt.h b/src/whereInt.h
index 558a1e3eb9..d790653a71 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -608,5 +608,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
#define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */
#define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */
#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */
+#define WHERE_CULLED 0x00800000 /* nOut reduced by extra WHERE terms */
#endif /* !defined(SQLITE_WHEREINT_H) */
From 7e910f6422553150a62332bdc2f3c21b16184abb Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 9 Dec 2021 01:28:15 +0000
Subject: [PATCH 15/18] Add SQLITE_TESTCTRL_LOGEST and enhance the LogEst
utility program. Improvements to testability of bloom filters.
FossilOrigin-Name: 88b43d798cc5aa59855e92d3e658aee9f0a5def6ffbc5db77af048d75ecdf8cc
---
manifest | 28 ++++++++++++++--------------
manifest.uuid | 2 +-
src/main.c | 20 ++++++++++++++++++++
src/sqlite.h.in | 3 ++-
src/sqliteInt.h | 2 --
src/util.c | 2 --
src/vdbe.c | 6 ++----
src/where.c | 34 +++++++++++++++++++---------------
src/whereInt.h | 2 +-
src/wherecode.c | 7 +++----
tool/logest.c | 5 ++++-
11 files changed, 66 insertions(+), 45 deletions(-)
diff --git a/manifest b/manifest
index 4c4a9d684d..7cbde1e303 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improvements\son\sthe\sdecision\sof\swhether\sor\snot\sto\suse\sa\sBloom\sfilter.
-D 2021-12-08T19:50:45.145
+C Add\sSQLITE_TESTCTRL_LOGEST\sand\senhance\sthe\sLogEst\sutility\sprogram.\nImprovements\sto\stestability\sof\sbloom\sfilters.
+D 2021-12-09T01:28:15.168
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -515,7 +515,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c
-F src/main.c 1ea70751e6005ab6a9f784730fa0919efaa6639440a287deb73cb711e5aae57a
+F src/main.c 674a0fdfc2808e1d5a78b2eefe2ec3f93428cf82f0f6c013d577df1a1caa5940
F src/malloc.c d9172a3946f11384f2fd6a799554ee26c6bb407c4bd0874a456ed485a2e362e4
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -553,10 +553,10 @@ F src/resolve.c 4a1db4aadd802683db40ca2dbbb268187bd195f10cbdb7206dbd8ac988795571
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c a7a3d9f54eb24821ec5f67f2e5589b68a5d42d46fc5849d7376886777d93a85a
F src/shell.c.in cda1eaf0292259b4b0721a5e03af9701fd482ebc37ce6a86ddc94cd9a38bb826
-F src/sqlite.h.in bb56040e3c498711c9f77727e477674395a50931ccba8095cfef5c8fb3c3e138
+F src/sqlite.h.in 50c8f27251b11f1c89b06abc6e4085fce15151bcbd355a44609ecb2ba5424841
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h f4fbb14ea32d57b813aabf82f586d2ac042234dd89df1c03281f557907745b98
+F src/sqliteInt.h 31b9673bc26b5b2a846bf26ce7124c869f64368f4eaac865d9350749ea314000
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -621,9 +621,9 @@ F src/trigger.c 2ef56f0b7b75349a5557d0604b475126329c2e1a02432e7d49c4c710613e8254
F src/update.c d6f5c7b9e072660757ac7d58175aca11c07cb95ebbb297ae7f38853700f52328
F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c 6dfbd0bd1954e9531e1c511e5d20390d7dab9ffbf1e20a37c960d1aaf8582b46
+F src/util.c 569349b0bddcbfbc661856f446adb92e1b0a47b3cbef548da9fc5aa639d7964c
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c fd8542b7131f299659871535a41ea732764fb25e4d2931965c97fa36658c50d7
+F src/vdbe.c 855ee903521fcc5a799f673f5b05fc599dc50a31e6cb6a15e1e8a6858087595b
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
F src/vdbeInt.h 910985ac2783fe0938b314d811759d53fd25caf215810f62ca1ff068d6d60d7b
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
@@ -638,9 +638,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 89958d4fc7c45e916882ebc97481d98597f516ce3d778ace3271aacf34e24e91
-F src/whereInt.h c2cb535e755b25a7e152bdb407cbb2f62bdb8747c44bf2d984139f5cbebb8704
-F src/wherecode.c e2207f011b7e5bdef5722da5e8d95eb30ad01051b3526757901ecb19a9e98ff3
+F src/where.c 4946af4e7d073fd35c97ffce4c4b4f3d84f6a0c9e5cb20342560cfaf072ef3a6
+F src/whereInt.h e83f7ba73db5b1b2685118fad67d178fbe04751a25419f0f6ff73e58b4807325
+F src/wherecode.c 560424f5845ec4ef1e36e8a6a066cc497aaf8163fef29fa18a0317e1e14ba9b7
F src/whereexpr.c 791544603b254cf11f8e84e3b50b0863c57322e9f213b828680f658e232ebc57
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1856,7 +1856,7 @@ F tool/lemon.c 258881835bd5bccd0c74fb110fe54244ff18e8e7ef3d949cbdab7187f02132bb
F tool/lempar.c 57478ea48420da05faa873c6d1616321caa5464644588c97fbe8e0ea04450748
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
-F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
+F tool/logest.c 83dbfda91615f1db5dce38215303d8bb456f437342d2c64262406dbdd1c931e2
F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439
F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176
F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d3250256772e3348abe887c0ca3550a6647cce3804c9456a9d0112aea7ee1c46
-R a7fca9e9fc2528f3e921fdfbeaa2e361
+P 0fb2a4e08f518cb38ea3edc6a084d1e4874fd622ba3cf9101b49b3e7dc1a3f2b
+R 13197c53db0503b249bb051590a0a6df
U drh
-Z 815ca39cf2b788796cb5ebc9be23c44d
+Z 80d7c2199cca2c3c9ca5b22286927a11
diff --git a/manifest.uuid b/manifest.uuid
index a9d9e689b3..2d8c59a7eb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0fb2a4e08f518cb38ea3edc6a084d1e4874fd622ba3cf9101b49b3e7dc1a3f2b
\ No newline at end of file
+88b43d798cc5aa59855e92d3e658aee9f0a5def6ffbc5db77af048d75ecdf8cc
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 804719f176..1b7853f609 100644
--- a/src/main.c
+++ b/src/main.c
@@ -4347,6 +4347,26 @@ int sqlite3_test_control(int op, ...){
break;
}
+ /* sqlite3_test_control(SQLITE_TESTCTRL_LOGEST,
+ ** double fIn, // Input value
+ ** int *pLogEst, // sqlite3LogEstFromDouble(fIn)
+ ** u64 *pInt, // sqlite3LogEstToInt(*pLogEst)
+ ** int *pLogEst2 // sqlite3LogEst(*pInt)
+ ** );
+ **
+ ** Test access for the LogEst conversion routines.
+ */
+ case SQLITE_TESTCTRL_LOGEST: {
+ double rIn = va_arg(ap, double);
+ LogEst rLogEst = sqlite3LogEstFromDouble(rIn);
+ u64 iInt = sqlite3LogEstToInt(rLogEst);
+ va_arg(ap, int*)[0] = rLogEst;
+ va_arg(ap, u64*)[0] = iInt;
+ va_arg(ap, int*)[0] = sqlite3LogEst(iInt);
+ break;
+ }
+
+
#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD)
/* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue)
**
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 4125122e17..f7cf4d2fe1 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -7944,7 +7944,8 @@ int sqlite3_test_control(int op, ...);
#define SQLITE_TESTCTRL_SEEK_COUNT 30
#define SQLITE_TESTCTRL_TRACEFLAGS 31
#define SQLITE_TESTCTRL_TUNE 32
-#define SQLITE_TESTCTRL_LAST 32 /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_LOGEST 33
+#define SQLITE_TESTCTRL_LAST 33 /* Largest TESTCTRL */
/*
** CAPI3REF: SQL Keyword Checking
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 90e60d30b0..0104bace6d 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4780,9 +4780,7 @@ int sqlite3Utf8CharLen(const char *pData, int nByte);
u32 sqlite3Utf8Read(const u8**);
LogEst sqlite3LogEst(u64);
LogEst sqlite3LogEstAdd(LogEst,LogEst);
-#ifndef SQLITE_OMIT_VIRTUALTABLE
LogEst sqlite3LogEstFromDouble(double);
-#endif
u64 sqlite3LogEstToInt(LogEst);
VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int);
const char *sqlite3VListNumToName(VList*,int);
diff --git a/src/util.c b/src/util.c
index d93c298116..8ea951fa16 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1586,7 +1586,6 @@ LogEst sqlite3LogEst(u64 x){
return a[x&7] + y - 10;
}
-#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Convert a double into a LogEst
** In other words, compute an approximation for 10*log2(x).
@@ -1601,7 +1600,6 @@ LogEst sqlite3LogEstFromDouble(double x){
e = (a>>52) - 1022;
return e*10;
}
-#endif /* SQLITE_OMIT_VIRTUALTABLE */
/*
** Convert a LogEst into an integer.
diff --git a/src/vdbe.c b/src/vdbe.c
index 0abe64a71f..ef60ed0123 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -8222,10 +8222,8 @@ case OP_Filter: { /* jump */
assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
pIn1 = &aMem[pOp->p1];
- if( (pIn1->flags & MEM_Blob)==0 || NEVER(pIn1->n<=0) ){
- VdbeBranchTaken(0, 2);
- break;
- }
+ assert( (pIn1->flags & MEM_Blob)!=0 );
+ assert( pIn1->n >= 1 );
h = filterHash(aMem, pOp);
#ifdef SQLITE_DEBUG
if( db->flags&SQLITE_VdbeTrace ){
diff --git a/src/where.c b/src/where.c
index 065d14fd64..859dac7cbb 100644
--- a/src/where.c
+++ b/src/where.c
@@ -1009,7 +1009,7 @@ static SQLITE_NOINLINE void constructBloomFilter(
do{
const SrcItem *pItem;
const Table *pTab;
- int sz;
+ u64 sz;
sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
addrCont = sqlite3VdbeMakeLabel(pParse);
iCur = pLevel->iTabCur;
@@ -1027,17 +1027,13 @@ static SQLITE_NOINLINE void constructBloomFilter(
assert( pItem!=0 );
pTab = pItem->pTab;
assert( pTab!=0 );
- if( pTab->tabFlags & TF_HasStat1 ){
- sz = sqlite3LogEstToInt(pItem->pTab->nRowLogEst);
- if( sz<10000 ){
- sz = 10000;
- }else if( sz>10000000 ){
- sz = 10000000;
- }
- }else{
+ sz = sqlite3LogEstToInt(pTab->nRowLogEst);
+ if( sz<10000 ){
sz = 10000;
+ }else if( sz>10000000 ){
+ sz = 10000000;
}
- sqlite3VdbeAddOp2(v, OP_Blob, sz, pLevel->regFilter);
+ sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
@@ -2503,7 +2499,11 @@ static void whereLoopOutputAdjust(
}
if( j<0 ){
if( pLoop->maskSelf==pTerm->prereqAll ){
- pLoop->wsFlags |= WHERE_CULLED;
+ /* If there are extra terms in the WHERE clause not used by an index
+ ** that depend only on the table being scanned, and that will tend to
+ ** cause many rows to be omitted, then mark that table as
+ ** "self-culling". */
+ pLoop->wsFlags |= WHERE_SELFCULL;
}
if( pTerm->truthProb<=0 ){
/* If a truth probability is specified using the likelihood() hints,
@@ -4974,10 +4974,13 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
**
** (1) The SEARCH happens more than N times where N is the number
** of rows in the table that is being considered for the Bloom
-** filter. (TO DO: Make this condition more precise.)
-** (2) Most searches are expected to find zero rows
+** filter.
+** (2) Some searches are expected to find zero rows. (This is determined
+** by the WHERE_SELFCULL flag on the term.)
** (3) The table being searched is not the right table of a LEFT JOIN
-** (4) Bloom-filter processing is not disabled
+** (4) Bloom-filter processing is not disabled. (Checked by the
+** caller.)
+** (5) The size of the table being searched is known by ANALYZE.
**
** This block of code merely checks to see if a Bloom filter would be
** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
@@ -4995,7 +4998,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
nSearch = pWInfo->a[0].pWLoop->nOut;
for(i=1; i<pWInfo->nLevel; i++){
WhereLoop *pLoop = pWInfo->a[i].pWLoop;
- const int reqFlags = (WHERE_CULLED|WHERE_COLUMN_EQ);
+ const int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
if( (pLoop->wsFlags & reqFlags)==reqFlags
&& (pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0
){
@@ -5004,6 +5007,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
pTab->tabFlags |= TF_StatsUsed;
if( nSearch > pTab->nRowLogEst
&& (pItem->fg.jointype & JT_LEFT)==0
+ && (pTab->tabFlags & TF_HasStat1)!=0
){
pLoop->wsFlags |= WHERE_BLOOMFILTER;
pLoop->wsFlags &= ~WHERE_IDX_ONLY;
diff --git a/src/whereInt.h b/src/whereInt.h
index d790653a71..8051b78a02 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -608,6 +608,6 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
#define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */
#define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */
#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */
-#define WHERE_CULLED 0x00800000 /* nOut reduced by extra WHERE terms */
+#define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */
#endif /* !defined(SQLITE_WHEREINT_H) */
diff --git a/src/wherecode.c b/src/wherecode.c
index 42cfcc9eac..05d1f12ced 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1393,13 +1393,12 @@ static SQLITE_NOINLINE void filterPullDown(
if( pLoop->prereq & notReady ) continue;
if( pLoop->wsFlags & WHERE_IPK ){
WhereTerm *pTerm = pLoop->aLTerm[0];
- int r1, regRowid;
+ int regRowid;
assert( pTerm!=0 );
assert( pTerm->pExpr!=0 );
testcase( pTerm->wtFlags & TERM_VIRTUAL );
- r1 = sqlite3GetTempReg(pParse);
- regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, r1);
- if( regRowid!=r1 ) sqlite3ReleaseTempReg(pParse, r1);
+ regRowid = sqlite3GetTempReg(pParse);
+ regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid);
sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter,
addrNxt, regRowid, 1);
VdbeCoverage(pParse->pVdbe);
diff --git a/tool/logest.c b/tool/logest.c
index e936e02cbe..580a3a6f96 100644
--- a/tool/logest.c
+++ b/tool/logest.c
@@ -75,6 +75,7 @@ static sqlite3_uint64 logEstToInt(LogEst x){
x /= 10;
if( n>=5 ) n -= 2;
else if( n>=1 ) n -= 1;
+ if( x>60 ) return ((sqlite3_uint64)0xffffffff)<<32 + 0xffffffff;
if( x>=3 ) return (n+8)<<(x-3);
return (n+8)>>(3-x);
}
@@ -149,7 +150,7 @@ int main(int argc, char **argv){
}else if( z[0]=='^' ){
a[n++] = (LogEst)atoi(z+1);
}else if( isInteger(z) ){
- a[n++] = logEstFromInteger(atoi(z));
+ a[n++] = logEstFromInteger(atoll(z));
}else if( isFloat(z) && z[0]!='-' ){
a[n++] = logEstFromDouble(atof(z));
}else{
@@ -161,6 +162,8 @@ int main(int argc, char **argv){
printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i]));
}else if( a[i]<10 ){
printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0);
+ }else if( a[i]>100 ){
+ printf("%5d (%lld)\n", a[i], logEstToInt(a[i]));
}else{
sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024;
printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100);
From 5d88be8f488e2915563cd823bbf632d167b7ca51 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 9 Dec 2021 16:17:43 +0000
Subject: [PATCH 16/18] Add ".mode off" and ".mode count" to the CLI.
FossilOrigin-Name: b11f4d080aa9e6f694e2ec401e871f42bf25997e8e8bf77fa9b6014a50466e3c
---
manifest | 12 ++++++------
manifest.uuid | 2 +-
src/shell.c.in | 18 +++++++++++++++++-
3 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/manifest b/manifest
index 7cbde1e303..9862becc43 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sSQLITE_TESTCTRL_LOGEST\sand\senhance\sthe\sLogEst\sutility\sprogram.\nImprovements\sto\stestability\sof\sbloom\sfilters.
-D 2021-12-09T01:28:15.168
+C Add\s".mode\soff"\sand\s".mode\scount"\sto\sthe\sCLI.
+D 2021-12-09T16:17:43.145
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -552,7 +552,7 @@ F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c
F src/resolve.c 4a1db4aadd802683db40ca2dbbb268187bd195f10cbdb7206dbd8ac988795571
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c a7a3d9f54eb24821ec5f67f2e5589b68a5d42d46fc5849d7376886777d93a85a
-F src/shell.c.in cda1eaf0292259b4b0721a5e03af9701fd482ebc37ce6a86ddc94cd9a38bb826
+F src/shell.c.in 239bee1085d94964f02582b0714dc3fc85cfc16e27e95813e4dbc24bb215a7e0
F src/sqlite.h.in 50c8f27251b11f1c89b06abc6e4085fce15151bcbd355a44609ecb2ba5424841
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0fb2a4e08f518cb38ea3edc6a084d1e4874fd622ba3cf9101b49b3e7dc1a3f2b
-R 13197c53db0503b249bb051590a0a6df
+P 88b43d798cc5aa59855e92d3e658aee9f0a5def6ffbc5db77af048d75ecdf8cc
+R 9caa791b08a3199a0c8bfc0ce172ce1a
U drh
-Z 80d7c2199cca2c3c9ca5b22286927a11
+Z 89026152bfb77d836155ab364d1ebd27
diff --git a/manifest.uuid b/manifest.uuid
index 2d8c59a7eb..274a6197ab 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-88b43d798cc5aa59855e92d3e658aee9f0a5def6ffbc5db77af048d75ecdf8cc
\ No newline at end of file
+b11f4d080aa9e6f694e2ec401e871f42bf25997e8e8bf77fa9b6014a50466e3c
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index c46eaffd62..3262f98c12 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1253,6 +1253,8 @@ struct ShellState {
#define MODE_Markdown 14 /* Markdown formatting */
#define MODE_Table 15 /* MySQL-style table formatting */
#define MODE_Box 16 /* Unicode box-drawing characters */
+#define MODE_Count 17 /* Output only a count of the rows of output */
+#define MODE_Off 18 /* No query output shown */
static const char *modeDescr[] = {
"line",
@@ -1271,7 +1273,9 @@ static const char *modeDescr[] = {
"json",
"markdown",
"table",
- "box"
+ "box",
+ "count",
+ "off"
};
/*
@@ -2093,6 +2097,10 @@ static int shell_callback(
if( azArg==0 ) return 0;
switch( p->cMode ){
+ case MODE_Count:
+ case MODE_Off: {
+ break;
+ }
case MODE_Line: {
int w = 5;
if( azArg==0 ) break;
@@ -3310,6 +3318,7 @@ static void exec_prepared_stmt(
sqlite3_stmt *pStmt /* Statment to run */
){
int rc;
+ sqlite3_uint64 nRow = 0;
if( pArg->cMode==MODE_Column
|| pArg->cMode==MODE_Table
@@ -3342,6 +3351,7 @@ static void exec_prepared_stmt(
azCols[i] = (char *)sqlite3_column_name(pStmt, i);
}
do{
+ nRow++;
/* extract the data and data types */
for(i=0; i<nCol; i++){
aiTypes[i] = x = sqlite3_column_type(pStmt, i);
@@ -3369,6 +3379,8 @@ static void exec_prepared_stmt(
sqlite3_free(pData);
if( pArg->cMode==MODE_Json ){
fputs("]\n", pArg->out);
+ }else if( pArg->cMode==MODE_Count ){
+ printf("%llu row%s\n", nRow, nRow!=1 ? "s" : "");
}
}
}
@@ -8879,6 +8891,10 @@ static int do_meta_command(char *zLine, ShellState *p){
p->mode = MODE_Table;
}else if( c2=='b' && strncmp(azArg[1],"box",n2)==0 ){
p->mode = MODE_Box;
+ }else if( c2=='c' && strncmp(azArg[1],"count",n2)==0 ){
+ p->mode = MODE_Count;
+ }else if( c2=='o' && strncmp(azArg[1],"off",n2)==0 ){
+ p->mode = MODE_Off;
}else if( c2=='j' && strncmp(azArg[1],"json",n2)==0 ){
p->mode = MODE_Json;
}else if( nArg==1 ){
From a11c5e22b5eb32cdf2061a6c19fd7bf4d2a36e05 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 9 Dec 2021 18:44:03 +0000
Subject: [PATCH 17/18] Enable bloom filters for the right table of a LEFT
JOIN. Fix unreachable branches.
FossilOrigin-Name: d342ab722de1fc3f34219c9755253db9d88eb50a6fa5cc257207008b00e7fc03
---
manifest | 16 ++++++-------
manifest.uuid | 2 +-
src/where.c | 4 ++--
src/wherecode.c | 63 +++++++++++++++++++++++--------------------------
test/join5.test | 1 +
5 files changed, 42 insertions(+), 44 deletions(-)
diff --git a/manifest b/manifest
index 9862becc43..a498dda5fc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\s".mode\soff"\sand\s".mode\scount"\sto\sthe\sCLI.
-D 2021-12-09T16:17:43.145
+C Enable\sbloom\sfilters\sfor\sthe\sright\stable\sof\sa\sLEFT\sJOIN.\s\sFix\sunreachable\nbranches.
+D 2021-12-09T18:44:03.070
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -638,9 +638,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 4946af4e7d073fd35c97ffce4c4b4f3d84f6a0c9e5cb20342560cfaf072ef3a6
+F src/where.c abd620b4823c0c72322ad5307805d0a350e808bd2424e3e00a1c3ecae32b8a09
F src/whereInt.h e83f7ba73db5b1b2685118fad67d178fbe04751a25419f0f6ff73e58b4807325
-F src/wherecode.c 560424f5845ec4ef1e36e8a6a066cc497aaf8163fef29fa18a0317e1e14ba9b7
+F src/wherecode.c c2324c3204b7f51648cc71aa0c98bff3c72c96c2e0c2169809a921febef7e5e1
F src/whereexpr.c 791544603b254cf11f8e84e3b50b0863c57322e9f213b828680f658e232ebc57
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1138,7 +1138,7 @@ F test/join.test 25da4f53523a4aa17c893134b47fba6aa4799bb33350517b157785878290e23
F test/join2.test 9bdc615841b91c97a16d68bad9508aea11fa0c6b34e5689847bcc4dac70e4990
F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
-F test/join5.test f418fccdfefa41f1659663463aa517431ddcf3e30ccbb80e64173b7d615a03f4
+F test/join5.test d22395f7d4020a58cabbc8316f300a5cfef84aee9e8ba7ce79b33cc43a3e1e2e
F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c
F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 88b43d798cc5aa59855e92d3e658aee9f0a5def6ffbc5db77af048d75ecdf8cc
-R 9caa791b08a3199a0c8bfc0ce172ce1a
+P b11f4d080aa9e6f694e2ec401e871f42bf25997e8e8bf77fa9b6014a50466e3c
+R e684e836076af2e815def08dd3898cfa
U drh
-Z 89026152bfb77d836155ab364d1ebd27
+Z 309930b0d62304a131a19ba2d8419e5a
diff --git a/manifest.uuid b/manifest.uuid
index 274a6197ab..c9a3436415 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b11f4d080aa9e6f694e2ec401e871f42bf25997e8e8bf77fa9b6014a50466e3c
\ No newline at end of file
+d342ab722de1fc3f34219c9755253db9d88eb50a6fa5cc257207008b00e7fc03
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 859dac7cbb..44423f4bda 100644
--- a/src/where.c
+++ b/src/where.c
@@ -5000,15 +5000,15 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
WhereLoop *pLoop = pWInfo->a[i].pWLoop;
const int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
if( (pLoop->wsFlags & reqFlags)==reqFlags
- && (pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0
+ && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
){
SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
Table *pTab = pItem->pTab;
pTab->tabFlags |= TF_StatsUsed;
if( nSearch > pTab->nRowLogEst
- && (pItem->fg.jointype & JT_LEFT)==0
&& (pTab->tabFlags & TF_HasStat1)!=0
){
+ testcase( pItem->fg.jointype & JT_LEFT );
pLoop->wsFlags |= WHERE_BLOOMFILTER;
pLoop->wsFlags &= ~WHERE_IDX_ONLY;
WHERETRACE(0xffff, (
diff --git a/src/wherecode.c b/src/wherecode.c
index 05d1f12ced..8cc76b1fdf 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -236,42 +236,37 @@ int sqlite3WhereExplainBloomFilter(
const WhereLevel *pLevel /* Bloom filter on this level */
){
int ret = 0;
-#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
- if( sqlite3ParseToplevel(pParse)->explain==2 )
-#endif
- {
- SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom];
- Vdbe *v = pParse->pVdbe; /* VM being constructed */
- sqlite3 *db = pParse->db; /* Database handle */
- char *zMsg; /* Text to add to EQP output */
- int i; /* Loop counter */
- WhereLoop *pLoop; /* The where loop */
- StrAccum str; /* EQP output string */
- char zBuf[100]; /* Initial space for EQP output string */
+ SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+ Vdbe *v = pParse->pVdbe; /* VM being constructed */
+ sqlite3 *db = pParse->db; /* Database handle */
+ char *zMsg; /* Text to add to EQP output */
+ int i; /* Loop counter */
+ WhereLoop *pLoop; /* The where loop */
+ StrAccum str; /* EQP output string */
+ char zBuf[100]; /* Initial space for EQP output string */
- sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
- str.printfFlags = SQLITE_PRINTF_INTERNAL;
- sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem);
- pLoop = pLevel->pWLoop;
- if( pLoop->wsFlags & WHERE_IPK ){
- const Table *pTab = pItem->pTab;
- if( pTab->iPKey>=0 ){
- sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName);
- }else{
- sqlite3_str_appendf(&str, "rowid=?");
- }
+ sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
+ str.printfFlags = SQLITE_PRINTF_INTERNAL;
+ sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem);
+ pLoop = pLevel->pWLoop;
+ if( pLoop->wsFlags & WHERE_IPK ){
+ const Table *pTab = pItem->pTab;
+ if( pTab->iPKey>=0 ){
+ sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName);
}else{
- for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
- const char *z = explainIndexColumnName(pLoop->u.btree.pIndex, i);
- if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
- sqlite3_str_appendf(&str, "%s=?", z);
- }
+ sqlite3_str_appendf(&str, "rowid=?");
+ }
+ }else{
+ for(i=pLoop->nSkip; i<pLoop->u.btree.nEq; i++){
+ const char *z = explainIndexColumnName(pLoop->u.btree.pIndex, i);
+ if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5);
+ sqlite3_str_appendf(&str, "%s=?", z);
}
- sqlite3_str_append(&str, ")", 1);
- zMsg = sqlite3StrAccumFinish(&str);
- ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
- pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
}
+ sqlite3_str_append(&str, ")", 1);
+ zMsg = sqlite3StrAccumFinish(&str);
+ ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
+ pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
return ret;
}
#endif /* SQLITE_OMIT_EXPLAIN */
@@ -1390,7 +1385,9 @@ static SQLITE_NOINLINE void filterPullDown(
WhereLevel *pLevel = &pWInfo->a[iLevel];
WhereLoop *pLoop = pLevel->pWLoop;
if( pLevel->regFilter==0 ) continue;
- if( pLoop->prereq & notReady ) continue;
+ /* ,--- Because constructBloomFilter() has will not have set
+ ** vvvvv--' pLevel->regFilter if this were true. */
+ if( NEVER(pLoop->prereq & notReady) ) continue;
if( pLoop->wsFlags & WHERE_IPK ){
WhereTerm *pTerm = pLoop->aLTerm[0];
int regRowid;
diff --git a/test/join5.test b/test/join5.test
index e2ff2f6c27..0ae4ca1127 100644
--- a/test/join5.test
+++ b/test/join5.test
@@ -303,6 +303,7 @@ do_eqp_test 7.4 {
} {
QUERY PLAN
|--SCAN t3
+ |--BLOOM FILTER ON t4 (x=?)
`--SEARCH t4 USING INDEX t4xz (x=?)
}
From 5a4ac1cc302388e8717f9873fc90e5b648a74eac Mon Sep 17 00:00:00 2001
From: drh <>
Date: Thu, 9 Dec 2021 19:42:52 +0000
Subject: [PATCH 18/18] Remove unused code and fix comments. Final cleanup
before merging.
FossilOrigin-Name: ce42039f5647b1f276acf5d9911528ecb47df1544a587def72c8cd6b2f664289
---
manifest | 22 +++++++++++-----------
manifest.uuid | 2 +-
src/malloc.c | 21 ---------------------
src/sqlite.h.in | 15 +++++++++------
src/sqliteInt.h | 4 ----
src/vdbe.c | 17 -----------------
src/where.c | 6 +++---
src/wherecode.c | 2 +-
8 files changed, 25 insertions(+), 64 deletions(-)
diff --git a/manifest b/manifest
index a498dda5fc..d3b9720825 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enable\sbloom\sfilters\sfor\sthe\sright\stable\sof\sa\sLEFT\sJOIN.\s\sFix\sunreachable\nbranches.
-D 2021-12-09T18:44:03.070
+C Remove\sunused\scode\sand\sfix\scomments.\s\sFinal\scleanup\sbefore\smerging.
+D 2021-12-09T19:42:52.951
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -516,7 +516,7 @@ F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c
F src/main.c 674a0fdfc2808e1d5a78b2eefe2ec3f93428cf82f0f6c013d577df1a1caa5940
-F src/malloc.c d9172a3946f11384f2fd6a799554ee26c6bb407c4bd0874a456ed485a2e362e4
+F src/malloc.c ef796bcc0e81d845d59a469f1cf235056caf9024172fd524e32136e65593647b
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75
@@ -553,10 +553,10 @@ F src/resolve.c 4a1db4aadd802683db40ca2dbbb268187bd195f10cbdb7206dbd8ac988795571
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c a7a3d9f54eb24821ec5f67f2e5589b68a5d42d46fc5849d7376886777d93a85a
F src/shell.c.in 239bee1085d94964f02582b0714dc3fc85cfc16e27e95813e4dbc24bb215a7e0
-F src/sqlite.h.in 50c8f27251b11f1c89b06abc6e4085fce15151bcbd355a44609ecb2ba5424841
+F src/sqlite.h.in 5999d6db0e65afbd686b76cddc385b310aa3815624edba43987913067f50e209
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h 31b9673bc26b5b2a846bf26ce7124c869f64368f4eaac865d9350749ea314000
+F src/sqliteInt.h b4391c3c2ae0a8020ce0f543fc2b529f9bcdf72ab7ba3c31d170e3228169162f
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -623,7 +623,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
F src/util.c 569349b0bddcbfbc661856f446adb92e1b0a47b3cbef548da9fc5aa639d7964c
F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c 855ee903521fcc5a799f673f5b05fc599dc50a31e6cb6a15e1e8a6858087595b
+F src/vdbe.c 5c05b1149a930851f36b29172002dfa56ce703e13f82d5bb4242a29dc144e350
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
F src/vdbeInt.h 910985ac2783fe0938b314d811759d53fd25caf215810f62ca1ff068d6d60d7b
F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
@@ -638,9 +638,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c abd620b4823c0c72322ad5307805d0a350e808bd2424e3e00a1c3ecae32b8a09
+F src/where.c 71bbbac8748ddd470dd3c4213342dce2e39c2c0aa84f9b03d80bbc075fb7c502
F src/whereInt.h e83f7ba73db5b1b2685118fad67d178fbe04751a25419f0f6ff73e58b4807325
-F src/wherecode.c c2324c3204b7f51648cc71aa0c98bff3c72c96c2e0c2169809a921febef7e5e1
+F src/wherecode.c 98ec56212ea7c0993f5eb808c515076d42c814f833c1641022927903cef0e811
F src/whereexpr.c 791544603b254cf11f8e84e3b50b0863c57322e9f213b828680f658e232ebc57
F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b11f4d080aa9e6f694e2ec401e871f42bf25997e8e8bf77fa9b6014a50466e3c
-R e684e836076af2e815def08dd3898cfa
+P d342ab722de1fc3f34219c9755253db9d88eb50a6fa5cc257207008b00e7fc03
+R 5df525a734b3c13f01d7f10e288ab697
U drh
-Z 309930b0d62304a131a19ba2d8419e5a
+Z ccda542cb10f0dae6599ca15d7e6a94e
diff --git a/manifest.uuid b/manifest.uuid
index c9a3436415..96b8bdf103 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d342ab722de1fc3f34219c9755253db9d88eb50a6fa5cc257207008b00e7fc03
\ No newline at end of file
+ce42039f5647b1f276acf5d9911528ecb47df1544a587def72c8cd6b2f664289
\ No newline at end of file
diff --git a/src/malloc.c b/src/malloc.c
index 9a2eaeec15..932cecc210 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -211,27 +211,6 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
return mx;
}
-#if 0
-/*
-** Return an estimate of the amount of unallocated memory.
-**
-** This the hard heap limit minus the current memory usage. It might
-** not be possible to allocate this much memory all at once. This is
-** only an estimate.
-*/
-sqlite3_int64 sqlite3EstMemoryAvailable(void){
- sqlite3_int64 n;
- sqlite3_mutex_enter(mem0.mutex);
- n = mem0.alarmThreshold;
- if( n<=0 ) n = mem0.hardLimit;
- sqlite3_mutex_leave(mem0.mutex);
- if( n<=0 ) n = LARGEST_INT64;
- n -= sqlite3_memory_used();
- if( n<0 ) n = 0;
- return n;
-}
-#endif
-
/*
** Trigger the alarm
*/
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f7cf4d2fe1..63a6f37e9a 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8468,12 +8468,15 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
** The counter is incremented on the first [sqlite3_step()] call of each
** cycle.
**
-** [[SQLITE_STMTSTATUS_FILTER HIT]] <dt>SQLITE_STMTSTATUS_FILTER_HIT</dt>
-** <dd>^This is the number of times that a join step was bypassed because
-** a Bloom filtered returned non-found. The corresponding
-** SQLITE_STMTSTATUS_FILTER_MISS value is the number of times that the
-** Bloom filter returned a find, and thus the join step had to be processed
-** as normal.
+** [[SQLITE_STMTSTATUS_FILTER_MISS]]
+** [[SQLITE_STMTSTATUS_FILTER HIT]]
+** <dt>SQLITE_STMTSTATUS_FILTER_HIT<br>
+** SQLITE_STMTSTATUS_FILTER_MISS</dt>
+** <dd>^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join
+** step was bypassed because a Bloom filter returned not-found. The
+** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of
+** times that the Bloom filter returned a find, and thus the join step
+** had to be processed as normal.
**
** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt>
** <dd>^This is the approximate number of bytes of heap memory
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 0104bace6d..bd02e2626f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3927,7 +3927,6 @@ struct Sqlite3Config {
int iOnceResetThreshold; /* When to reset OP_Once counters */
u32 szSorterRef; /* Min size in bytes to use sorter-refs */
unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */
- int iEstCountScale; /* Multiple RowCountEst() by this amount */
/* vvvv--- must be last ---vvv */
#ifdef SQLITE_DEBUG
sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */
@@ -4294,9 +4293,6 @@ void sqlite3MemSetDefault(void);
void sqlite3BenignMallocHooks(void (*)(void), void (*)(void));
#endif
int sqlite3HeapNearlyFull(void);
-#if 0
-sqlite3_int64 sqlite3EstMemoryAvailable(void);
-#endif
/*
** On systems with ample stack space and that support alloca(), make
diff --git a/src/vdbe.c b/src/vdbe.c
index ef60ed0123..27e030ec19 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -671,23 +671,6 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){
}
}
-/*
-** The minimum size (in bytes) for a Bloom filter.
-**
-** No Bloom filter will be smaller than this many bytes. But they
-** may be larger.
-*/
-#ifndef SQLITE_BLOOM_MIN
-# define SQLITE_BLOOM_MIN 10000
-#endif
-
-/*
-** The maximum size (in bytes) for a Bloom filter.
-*/
-#ifndef SQLITE_BLOOM_MAX
-# define SQLITE_BLOOM_MAX 1000000
-#endif
-
/*
** Compute a bloom filter hash using pOp->p4.i registers from aMem[] beginning
** with pOp->p3. Return the hash.
diff --git a/src/where.c b/src/where.c
index 44423f4bda..be75101b26 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4977,10 +4977,9 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
** filter.
** (2) Some searches are expected to find zero rows. (This is determined
** by the WHERE_SELFCULL flag on the term.)
-** (3) The table being searched is not the right table of a LEFT JOIN
-** (4) Bloom-filter processing is not disabled. (Checked by the
+** (3) Bloom-filter processing is not disabled. (Checked by the
** caller.)
-** (5) The size of the table being searched is known by ANALYZE.
+** (4) The size of the table being searched is known by ANALYZE.
**
** This block of code merely checks to see if a Bloom filter would be
** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
@@ -5000,6 +4999,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
WhereLoop *pLoop = pWInfo->a[i].pWLoop;
const int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
if( (pLoop->wsFlags & reqFlags)==reqFlags
+ /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
&& ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
){
SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
diff --git a/src/wherecode.c b/src/wherecode.c
index 8cc76b1fdf..dd73bc4e83 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1371,7 +1371,7 @@ static void whereApplyPartialIndexConstraints(
**
** We know that an inner loop uses a Bloom filter because it has the
** WhereLevel.regFilter set. If an inner-loop Bloom filter is checked,
-** then clear the WhereLoeve.regFilter value to prevent the Bloom filter
+** then clear the WhereLevel.regFilter value to prevent the Bloom filter
** from being checked a second time when the inner loop is evaluated.
*/
static SQLITE_NOINLINE void filterPullDown(