From be341502dbb10ca5941f41ce65df11c2d52290e3 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 25 Mar 2022 01:23:37 +0000
Subject: [PATCH] Fix the Bloom-filter optimization so that it does not use IS
 NULL or IS NOT NULL constraints from the WHERE clause when operating on a
 LEFT JOIN. [forum:/forumpost/031e262a89b6a9d2|Forum thread 031e262a89b6a9d2].

FossilOrigin-Name: 1a6328f2a5b4973094e5f85787145d652119822c86ec01a61f3f985c9d2903f2
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/where.c     | 13 +++++++++++--
 test/join5.test | 34 ++++++++++++++++++++++++++++++++++
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 5606959056..9769465203 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\s"PRAGMA\swritable_schema=ON"\sflag\sshould\snot\sallow\sOOM\serrors\sto\spass\nwhile\sparsing\sthe\sschema.\ndbsqlfuzz\s9cc49e1a53e1cef8e3a1496a88c683aa20483163.
-D 2022-03-24T14:01:55.718
+C Fix\sthe\sBloom-filter\soptimization\sso\sthat\sit\sdoes\snot\suse\sIS\sNULL\sor\nIS\sNOT\sNULL\sconstraints\sfrom\sthe\sWHERE\sclause\swhen\soperating\son\sa\sLEFT\sJOIN.\n[forum:/forumpost/031e262a89b6a9d2|Forum\sthread\s031e262a89b6a9d2].
+D 2022-03-25T01:23:37.631
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -639,7 +639,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 9cba7b9e58d4d07f08bb0cb5b9b982d359c8051050df9114ee927cf6798e5509
+F src/where.c 7c2c5feb3600d02b827effece817adda0cc1d96ddb9ebe7494c990d53dc26180
 F src/whereInt.h 15d2975c3b4c193c78c26674400a840da8647fe1777ae3b026e2d15937b38a03
 F src/wherecode.c 555f598a9ddad81761f084710fdb4f8733fe31bc14cd6b19f8ca4274a7eaa04c
 F src/whereexpr.c 2a71f5491798460c9590317329234d332d9eb1717cba4f3403122189a75c465e
@@ -1144,7 +1144,7 @@ F test/join.test 25cf0ac11c3b81fedfd166f9062166bdb39dea92f5a7c16cacbf6dc1f7f6702
 F test/join2.test 9bdc615841b91c97a16d68bad9508aea11fa0c6b34e5689847bcc4dac70e4990
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
-F test/join5.test 3d51c4ae5f1f373a03164ca3c88d438f64bed7a2c01cf69810e2ca3d0e9071c8
+F test/join5.test 0d63c7e43b3160b9d4b93f196ef83b6efc7751b9edd0d18c53a46fbec7a49cfc
 F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c
 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
@@ -1945,8 +1945,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 43143ad131f17734fd2eff849e0a1bc2e26daf6a28c7e07d697d38732e6af5fc
-R c6ecdf0c09fcfac1e33c997c57a43f56
+P a7abb725c7c783331e73837597917e25e5ff692aac8ec97a0a8c0e30db62d68b
+R 7df72da8b94385b7c430ac7b857fb766
 U drh
-Z 9ff38d7c1e3943096d040996e86f8218
+Z f44c9fa4c137748740188f91b08447a0
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 515843110e..2c894bd6bf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a7abb725c7c783331e73837597917e25e5ff692aac8ec97a0a8c0e30db62d68b
\ No newline at end of file
+1a6328f2a5b4973094e5f85787145d652119822c86ec01a61f3f985c9d2903f2
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index bec77cc45d..c8a3cba2ec 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2596,8 +2596,17 @@ static void whereLoopOutputAdjust(
         /* If there are extra terms in the WHERE clause not used by an index
         ** that depend only on the table being scanned, and that will tend to
         ** cause many rows to be omitted, then mark that table as
-        ** "self-culling". */
-        pLoop->wsFlags |= WHERE_SELFCULL;
+        ** "self-culling".
+        **
+        ** 2022-03-24:  Self-culling only applies if either the extra terms
+        ** are straight comparison operators that are non-true with NULL
+        ** operand, or if the loop is not a LEFT JOIN.
+        */
+        if( (pTerm->eOperator & 0x3f)!=0
+         || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0
+        ){
+          pLoop->wsFlags |= WHERE_SELFCULL;
+        }
       }
       if( pTerm->truthProb<=0 ){
         /* If a truth probability is specified using the likelihood() hints,
diff --git a/test/join5.test b/test/join5.test
index 518fab7a92..a5cedd8cb2 100644
--- a/test/join5.test
+++ b/test/join5.test
@@ -300,6 +300,13 @@ do_execsql_test 7.3 {
 
 do_eqp_test 7.4 {
   SELECT * FROM t3 LEFT JOIN t4 ON (t4.x = t3.x) WHERE (t4.y = ? OR t4.z = ?);
+} {
+  QUERY PLAN
+  |--SCAN t3
+  `--SEARCH t4 USING INDEX t4xz (x=?)
+} 
+do_eqp_test 7.4b {
+  SELECT * FROM t3 CROSS JOIN t4 ON (t4.x = t3.x) WHERE (+t4.y = ? OR t4.z = ?);
 } {
   QUERY PLAN
   |--SCAN t3
@@ -360,5 +367,32 @@ do_execsql_test 10.1 {
   SELECT x FROM v3; 
 } {0}
 
+# 2022-03-24 https://sqlite.org/forum/forumpost/031e262a89b6a9d2
+# Bloom-filter on a LEFT JOIN with NULL-based WHERE constraints.
+#
+reset_db
+do_execsql_test 11.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT);
+  CREATE TABLE t2(c INTEGER PRIMARY KEY, d INT);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<8)
+  INSERT INTO t1(a,b) SELECT x, 10*x FROM c;
+  INSERT INTO t2(c,d) SELECT b*2, 100*a FROM t1;
+  ANALYZE;
+  DELETE FROM sqlite_stat1;
+  INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES
+    ('t1',NULL,150105),('t2',NULL,98747);
+  ANALYZE sqlite_schema;
+} {}
+do_execsql_test 11.2 {
+  SELECT count(*) FROM t1 LEFT JOIN t2 ON c=b WHERE d IS NULL;
+} {4}
+do_execsql_test 11.3 {
+  SELECT count(*) FROM t1 LEFT JOIN t2 ON c=b WHERE d=100;
+} {1}
+do_execsql_test 11.4 {
+  SELECT count(*) FROM t1 LEFT JOIN t2 ON c=b WHERE d>=300;
+} {2}
+
+
 
 finish_test