Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens.

FossilOrigin-Name: b9fb69e55bb05a8819688ee63987f2a05c33d59b

ext/fts3/fts3.c

@@ -3572,11 +3572,16 @@ static void fts3EvalTokenCosts(
       }
     }else if( pExpr->eType!=FTSQUERY_NOT ){
       if( pExpr->eType==FTSQUERY_OR ){
-        pRoot = pExpr;
+        pRoot = pExpr->pLeft;
-        **ppOr = pExpr;
+        **ppOr = pRoot;
         (*ppOr)++;
       }
       fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
+      if( pExpr->eType==FTSQUERY_OR ){
+        pRoot = pExpr->pRight;
+        **ppOr = pRoot;
+        (*ppOr)++;
+      }
       fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
     }
   }
@@ -3736,7 +3741,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
     Fts3Expr **apOr;
     aTC = (Fts3TokenAndCost *)sqlite3_malloc(
         sizeof(Fts3TokenAndCost) * nToken
-      + sizeof(Fts3Expr *) * nOr
+      + sizeof(Fts3Expr *) * nOr * 2
     );
     apOr = (Fts3Expr **)&aTC[nToken];
 
@@ -4310,7 +4315,7 @@ int sqlite3Fts3EvalPhraseStats(
   int rc = SQLITE_OK;
   int iCol;
 
-  if( pExpr->bDeferred ){
+  if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
     assert( pCsr->nDoc>0 );
     for(iCol=0; iCol<pTab->nColumn; iCol++){
       aiOut[iCol*3 + 1] = pCsr->nDoc;

manifest

@@ -1,5 +1,5 @@
-C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations.
+C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens.
-D 2011-06-13T13:48:36.083
+D 2011-06-13T17:00:12.630
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a
+F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6
 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe
@@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9
 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
-F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559
+F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c
 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
@@ -466,7 +466,7 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
 F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35
-F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989
+F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
@@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d
-P 5f6b87f420f21749aa7c72e020c50aca74890086
+P e4ab6cdb101bbeb804820425cf569ee7dc2397fb
-R ca51faeb6c7aef68bca6b7079ff7ebda
+R 7fcdbf104822a42776783c2cd9e89101
 U dan
-Z 9a71ae3656c163debcbb9f929d0e7d22
+Z 0999cf6b26be67f673d9db0918d86701

manifest.uuid

@@ -1 +1 @@
-e4ab6cdb101bbeb804820425cf569ee7dc2397fb
+b9fb69e55bb05a8819688ee63987f2a05c33d59b

test/fts3auto.test

@@ -93,12 +93,45 @@ proc get_single_near_results {tbl expr arrayvar nullvar} {
     set iCol [lindex [split $key ,] 1]
     set iPhrase 0
     foreach c $counts($key) {
-      if {$c>0} { incr nHit($iPhrase,$iCol) 1 }
+      if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
-      incr nDoc($iPhrase,$iCol) $c
+      incr nHit($iPhrase,$iCol) $c
       incr iPhrase
     }
   }
 
+  if {[info exists ::fts3_deferred] && [llength $expr]==1} {
+    set phrase [lindex $expr 0]
+    set rewritten [list]
+    set partial 0
+    foreach tok $phrase {
+      if {[lsearch $::fts3_deferred $tok]>=0} {
+        lappend rewritten *
+      } else {
+        lappend rewritten $tok
+        set partial 1
+      }
+    }
+    if {$partial==0} {
+      set tblsize [db one "SELECT count(*) FROM $tbl"]
+      for {set iCol 0} {$iCol < $nCol} {incr iCol} {
+        set nHit(0,$iCol) $tblsize
+        set nDoc(0,$iCol) $tblsize
+      }
+    } elseif {$rewritten != $phrase} {
+      while {[lindex $rewritten end] == "*"} {
+        set rewritten [lrange $rewritten 0 end-1]
+      }
+      while {[lindex $rewritten 0] == "*"} {
+        set rewritten [lrange $rewritten 1 end]
+      }
+      get_single_near_results $tbl [list $rewritten] aRewrite nullentry
+      foreach docid [array names hits] {
+        set aMatchinfo($docid) $aRewrite($docid)
+      }
+      return
+    }
+  }
+
   # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
   # contain the output of matchinfo('x') for the document.
   #
@@ -107,8 +140,8 @@ proc get_single_near_results {tbl expr arrayvar nullvar} {
     for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
       for {set iCol 0} {$iCol<$nCol} {incr iCol} {
         lappend mi [lindex $counts($docid,$iCol) $iPhrase]
-        lappend mi $nDoc($iPhrase,$iCol)
         lappend mi $nHit($iPhrase,$iCol)
+        lappend mi $nDoc($iPhrase,$iCol)
       }
     }
     set aMatchinfo($docid) $mi
@@ -119,7 +152,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} {
   set nullentry [list]
   for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
     for {set iCol 0} {$iCol<$nCol} {incr iCol} {
-      lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol)
+      lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
     }
   }
 }
@@ -219,7 +252,7 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} {
   }
 }
 
-proc do_near_test {tn tbl expr} {
+proc do_fts3query_test {tn tbl expr} {
 
   get_near_results $tbl $expr aMatchinfo
   set match $expr
@@ -286,19 +319,20 @@ test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}
 
 #--------------------------------------------------------------------------
 # Test cases fts3auto-2.* run some simple tests using the 
-# [do_near_test] proc.
+# [do_fts3query_test] proc.
 #
 foreach {tn create} {
-  1    "CREATE VIRTUAL TABLE t1 USING fts4(a, b)"
+  1    "fts4(a, b)"
-  2    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)"
+  2    "fts4(a, b, order=DESC)"
-  3    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)"
+  3    "fts4(a, b, order=ASC)"
-  4    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)"
+  4    "fts4(a, b, prefix=1)"
-  5    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)"
+  5    "fts4(a, b, order=DESC, prefix=1)"
-  6    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)"
+  6    "fts4(a, b, order=ASC, prefix=1)"
 } {
+break
   do_test 2.$tn.1 {
     catchsql { DROP TABLE t1 }
-    execsql  $create
+    execsql  "CREATE VIRTUAL TABLE t1 USING $create"
     for {set i 0} {$i<32} {incr i} {
       set doc [list]
       if {$i&0x01} {lappend doc one}
@@ -309,6 +343,7 @@ foreach {tn create} {
       execsql { INSERT INTO t1 VALUES($doc, null) }
     }
   } {}
+
   foreach {tn2 expr} {
     1     {one}
     2     {one NEAR/1 five}
@@ -327,7 +362,69 @@ foreach {tn create} {
     15    {"one two"}
     16    {"one two" NOT "three four"}
   } {
-    do_near_test 2.$tn.2.$tn2 t1 $expr
+    do_fts3query_test 2.$tn.2.$tn2 t1 $expr
+  }
+}
+
+#--------------------------------------------------------------------------
+# Some test cases involving deferred tokens.
+#
+proc make_token_deferrable {tbl token} {
+  set nRow [db one "SELECT count(*) FROM $tbl"]
+  set pgsz [db one "PRAGMA page_size"]
+
+  execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')"
+  execsql BEGIN
+  for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
+    set doc [string repeat "$token " 100]
+    execsql "INSERT INTO $tbl VALUES(\$doc)"
+  }
+  execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')"
+  execsql COMMIT
+}
+
+foreach {tn create} {
+  1    "fts4(x)"
+  2    "fts4(x, order=DESC)"
+} {
+  catchsql { DROP TABLE t1 }
+  execsql  "CREATE VIRTUAL TABLE t1 USING $create"
+  do_execsql_test 3.$tn.1 {
+    INSERT INTO t1 VALUES('a b c d e f g h i j k');
+    INSERT INTO t1 VALUES('b c d e f g h i j k a');
+    INSERT INTO t1 VALUES('c d e f g h i j k a b');
+    INSERT INTO t1 VALUES('d e f g h i j k a b c');
+    INSERT INTO t1 VALUES('e f g h i j k a b c d');
+    INSERT INTO t1 VALUES('f g h i j k a b c d e');
+    INSERT INTO t1 VALUES('a c e g i k');
+    INSERT INTO t1 VALUES('a d g j');
+    INSERT INTO t1 VALUES('c a b');
+  }
+
+  make_token_deferrable t1 c
+
+  set ::fts3_deferred [list]
+  foreach {tn2 expr} {
+    1     {a OR c}
+  } {
+    do_fts3query_test 3.$tn.2.$tn2 t1 $expr
+  }
+
+  set ::fts3_deferred [list c]
+  execsql { 
+    UPDATE t1_segments 
+    SET block = zeroblob(length(block)) 
+    WHERE length(block)>10000 AND 0
+  }
+  foreach {tn2 expr} {
+    1     {a NEAR c}
+    2     {a AND c}
+    3     {"a c"}
+    4     {"c a"}
+    5     {"a c" NEAR/1 g}
+    6     {"a c" NEAR/0 g}
+  } {
+    do_fts3query_test 3.$tn.2.$tn2 t1 $expr
   }
 }
 

test/fts3defer2.test

@@ -56,14 +56,13 @@ do_execsql_test 1.2.1 {
   SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
 } {{a b c d e f a x y}}
 
-breakpoint
 do_execsql_test 1.2.2 {
   SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
   FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
 } [list                              \
    {a b c d [e] [f] [a] x y}         \
    {0 1 8 1 0 0 10 1 0 2 12 1}       \
-   [list 3 1   1 1 1   1 1 1   1 8 8   8 5001 9]
+   [list 3 1   1 1 1   1 1 1   1 1 1   8 5001 9]
 ]
 
 do_execsql_test 1.2.3 {
@@ -72,7 +71,7 @@ do_execsql_test 1.2.3 {
 } [list                                 \
    {[a] b c d [e] [f] [a] x y}          \
    {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
-   [list 3 1   1 1 1   1 1 1   2 8 8   8 5001 9]
+   [list 3 1   1 1 1   1 1 1   2 2 1   8 5001 9]
 ]
 
 do_execsql_test 1.3.1 { DROP TABLE t1 }