Improve test coverage of fts3.c.

FossilOrigin-Name: 56b6432f8622d53ffd3a4d9a2244114f8531ed71

ext/fts3/fts3.c

@@ -381,13 +381,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
 int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
   const unsigned char *q = (const unsigned char *) p;
   sqlite_uint64 x = 0, y = 1;
-  while( (*q & 0x80) == 0x80 ){
+  while( (*q&0x80)==0x80 && q-(unsigned char *)p<FTS3_VARINT_MAX ){
     x += y * (*q++ & 0x7f);
     y <<= 7;
-    if( q - (unsigned char *)p >= FTS3_VARINT_MAX ){  /* bad data */
-      assert( 0 );
-      return 0;
-    }
   }
   x += y * (*q++);
   *v = (sqlite_int64) x;
@@ -402,7 +398,6 @@ int sqlite3Fts3GetVarint32(const char *p, int *pi){
  sqlite_int64 i;
  int ret = sqlite3Fts3GetVarint(p, &i);
  *pi = (int) i;
- assert( *pi==i );
  return ret;
 }
 
@@ -433,7 +428,7 @@ int sqlite3Fts3VarintLen(sqlite3_uint64 v){
 **     `mno`   becomes   mno
 */
 void sqlite3Fts3Dequote(char *z){
-  int quote;
+  char quote;
   int i, j;
 
   quote = z[0];
@@ -444,19 +439,22 @@ void sqlite3Fts3Dequote(char *z){
     case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
     default:    return;
   }
-  for(i=1, j=0; z[i]; i++){
+
+  i = 1;
+  j = 0;
+  while( ALWAYS(z[i]) ){
     if( z[i]==quote ){
       if( z[i+1]==quote ){
-        z[j++] = (char)quote;
+        z[j++] = quote;
-        i++;
+        i += 2;
       }else{
-        z[j++] = 0;
         break;
       }
     }else{
-      z[j++] = z[i];
+      z[j++] = z[i++];
     }
   }
+  z[j] = 0;
 }
 
 static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
@@ -891,11 +889,15 @@ static int fts3CursorSeek(Fts3Cursor *pCsr){
     if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
       return SQLITE_OK;
     }else{
-      int rc;
+      int rc = sqlite3_reset(pCsr->pStmt);
-      pCsr->isEof = 1;
+      if( rc==SQLITE_OK ){
-      if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){
+        /* If no row was found and no error has occured, then the %_content
-        rc = SQLITE_ERROR;
+        ** table is missing a row that is present in the full-text index.
+        ** The data structures are corrupt.
+        */
+        rc = SQLITE_CORRUPT;
       }
+      pCsr->isEof = 1;
       return rc;
     }
   }else{

ext/fts3/fts3_write.c

@@ -280,7 +280,7 @@ int sqlite3Fts3ReadBlock(
     sqlite3_bind_int64(pStmt, 1, iBlock);
     rc = sqlite3_step(pStmt); 
     if( rc!=SQLITE_ROW ){
-      return SQLITE_CORRUPT;
+      return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
     }
   
     *pnBlock = sqlite3_column_bytes(pStmt, 0);

manifest

@@ -1,5 +1,5 @@
-C Mark\sfts3ReallocOrFree\sand\sfts3InitVtab\sas\sstatic.\sTicket\s[ff44d82f3b].
+C Improve\stest\scoverage\sof\sfts3.c.
-D 2009-12-09T05:30:37
+D 2009-12-09T14:39:41
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -56,7 +56,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c cb9bd52e205b0f2f25fda5e166503f317f135731
+F ext/fts3/fts3.c 6e24afa93ecb3395b0e9467ca2b44fa7f66f4fdc
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 36f8e6d6fafa6d71a48e1810095e1e58fd55b199
 F ext/fts3/fts3_expr.c fcf6812dbfd9cb9a2cabaf50e741411794f83e7e
@@ -68,7 +68,7 @@ F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496
 F ext/fts3/fts3_tokenizer.c 3dc76eaea6b58ecfbe50135b8473aa668d712dcd
 F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
 F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
-F ext/fts3/fts3_write.c 643be9c7b097f5ea8189cf6ed840edd59485c45f
+F ext/fts3/fts3_write.c c7d549ac5a2733d1d3004a784ad8387f5c65fc75
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@@ -326,7 +326,7 @@ F test/descidx3.test 3394ad4d089335cac743c36a14129d6d931c316f
 F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
 F test/e_fkey.test fd1fcf89badd5f2773d7ac04775b5ff3488eda17
-F test/e_fts3.test 3ad57e7b920c7dcfa373dc2ed2105f83e1cf69a6
+F test/e_fts3.test 3acfdc1c363f245e6e251e067a5c375dd5e34731
 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea
 F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398
 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041
@@ -377,7 +377,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682
 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
 F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
 F test/fts3.test ae0433b09b12def08105640e57693726c4949338
-F test/fts3_common.tcl 31935839b1b601a5955572cb4e8060513c96bde0
+F test/fts3_common.tcl 8f75c15fa4507d923a1c21152e2716d77dc74661
 F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
 F test/fts3ac.test 356280144a2c92aa7b11474afadfe62a437fcd69
@@ -402,7 +402,7 @@ F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3malloc.test d02ee86b21edd2b43044e0d6dfdcd26cb6efddcb
 F test/fts3near.test dc196dd17b4606f440c580d45b3d23aa975fd077
-F test/fts3rnd.test ec82795eb358b7a4d6ce79e764d8d55556197584
+F test/fts3rnd.test 44fa7209327deabc3ca22e2ff7a293b0bd46c37f
 F test/func.test af106ed834001738246d276659406823e35cde7b
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/fuzz.test a4174c3009a3e2c2e14b31b364ebf7ddb49de2c9
@@ -777,7 +777,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P fdfdc777c59a108496c7381c8b267b38485e2c13
+P a9038306c33c88120d8bef27209d8f0641c85c9b
-R 3bbf1578efaf46ed0df1e1e874fb62b3
+R db340fa6c4afcd8bc8679373cdbca6c1
 U dan
-Z 73fc6e587a1afaf349e955a71f230978
+Z eeb82c42887166b9ef7e8ca6456b81ff

manifest.uuid

@@ -1 +1 @@
-a9038306c33c88120d8bef27209d8f0641c85c9b
+56b6432f8622d53ffd3a4d9a2244114f8531ed71

test/e_fts3.test

@@ -37,6 +37,9 @@ proc error_test {tn sql result} {
   uplevel [list do_error_test e_fts3-$tn $sql $result]
 }
 
+
+if 1 {
+
 #-------------------------------------------------------------------------
 # The body of the following [foreach] block contains test cases to verify
 # that the example code in fts3.html works as expected. The tests run three
@@ -587,10 +590,13 @@ read_test 7.3.4 {
   SELECT * FROM t7, t8 WHERE a MATCH 'number' AND b MATCH 'letter'
 } {{number four} {letter D} {number four} {letter E} {number five} {letter D} {number five} {letter E}}
 
+}
+
 #-------------------------------------------------------------------------
 # Test the quoting of FTS3 table column names. Names may be quoted using
 # any of "", '', ``` or [].
 #
+set DO_MALLOC_TEST 0
 ddl_test  8.1.1 { CREATE VIRTUAL TABLE t9a USING fts3("c1", [c2]) }
 ddl_test  8.1.2 { CREATE VIRTUAL TABLE t9b USING fts3('c1', `c2`) }
 read_test 8.1.3 { PRAGMA table_info(t9a) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0}
@@ -629,6 +635,46 @@ foreach DO_MALLOC_TEST {0 1 2} {
     SELECT * FROM t11 WHERE x MATCH 'rename*'
   } {{are renameable}}
 }
+#-------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------
+# Test a couple of cases involving corrupt data structures:
+#
+#   1) A case where a document referenced by the full-text index is
+#      not present in the %_content table.
+#
+#   2) A badly formatted b-tree segment node.
+#
+set DO_MALLOC_TEST 0
+ddl_test   10.1.1 { CREATE VIRTUAL TABLE ta USING fts3 }
+write_test 10.1.2 ta_content { 
+  INSERT INTO ta VALUES('During a summer vacation in 1790') }
+write_test 10.1.3 ta_content {
+  INSERT INTO ta VALUES('Wordsworth went on a walking tour') }
+write_test 10.1.4 ta_content { DELETE FROM ta_content WHERE rowid = 2 }
+read_test  10.1.5 {
+  SELECT * FROM ta WHERE ta MATCH 'summer'
+} {{During a summer vacation in 1790}}
+error_test 10.1.6 {
+  SELECT * FROM ta WHERE ta MATCH 'walking'
+} {database disk image is malformed}
+
+write_test 10.2.1 ta_content { DELETE FROM ta }
+write_test 10.2.2 ta_content { 
+  INSERT INTO ta VALUES('debate demonstrated the rising difficulty') }
+write_test 10.2.3 ta_content { 
+  INSERT INTO ta VALUES('Google released its browser beta') }
+
+set blob [db one {SELECT root FROM ta_segdir WHERE rowid = 2}]
+binary scan $blob "a6 a3 a*" start middle end
+set middle "\x0E\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x06\x06"
+set blob [binary format "a6 a* a*" $start $middle $end]
+write_test 10.2.4 ta_segdir { 
+  UPDATE ta_segdir SET root = $blob WHERE rowid = 2
+}
+error_test 10.2.5 {
+  SELECT * FROM ta WHERE ta MATCH 'beta'
+} {database disk image is malformed}
 
 
 finish_test

test/fts3_common.tcl

@@ -310,11 +310,11 @@ proc fts3_read {tbl where varname} {
 # match the expected results passed via parameter $result.
 #
 proc do_select_test {name sql result} {
-  doPassiveTest $name $sql [list 0 $result]
+  uplevel [list doPassiveTest $name $sql [list 0 $result]]
 }
 
 proc do_error_test {name sql error} {
-  doPassiveTest $name $sql [list 1 $error]
+  uplevel [list doPassiveTest $name $sql [list 1 $error]]
 }
 
 proc doPassiveTest {name sql catchres} {
@@ -333,7 +333,7 @@ proc doPassiveTest {name sql catchres} {
     for {set iFail 1} 1 {incr iFail} {
       if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
 
-      set res [catchsql $sql]
+      set res [uplevel [list catchsql $sql]]
       if {[lsearch -exact $answers $res]>=0} {
         set res $str
       }

test/fts3rnd.test

@@ -148,7 +148,14 @@ foreach nodesize {50 500 1000 2000} {
     for {set i 0} {$i < 100} {incr i} { insert_row $i }
   }
   
-  for {set iTest 1} {$iTest <= 100} {incr iTest} {
+  for {set iTest 0} {$iTest <= 100} {incr iTest} {
+
+    set DO_MALLOC_TEST 0
+    set nRep 10
+    if {$iTest==100 && $nodesize==50} { 
+      set DO_MALLOC_TEST 1 
+      set nRep 2
+    }
   
     # Delete one row, update one row and insert one row.
     #
@@ -176,8 +183,8 @@ foreach nodesize {50 500 1000 2000} {
     #
     for {set i 0} {$i < 10} {incr i} {
       set term [random_term]
-      do_test fts3rnd-1.$nodesize.$iTest.1.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.1.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $term }
+        SELECT docid FROM t1 WHERE t1 MATCH $term
       } [simple_phrase $term]
     }
 
@@ -185,66 +192,66 @@ foreach nodesize {50 500 1000 2000} {
     # to query for. Test that querying the Tcl array produces the same results
     # as querying the FTS3 table for the prefix.
     #
-    for {set i 0} {$i < 10} {incr i} {
+    for {set i 0} {$i < $nRep} {incr i} {
       set prefix [string range [random_term] 0 1]
       set match "${prefix}*"
-      do_test fts3rnd-1.$nodesize.$iTest.2.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+        SELECT docid FROM t1 WHERE t1 MATCH $match
       } [simple_phrase $match]
     }
 
     # Similar to the above, except for phrase queries.
     #
-    for {set i 0} {$i < 10} {incr i} {
+    for {set i 0} {$i < $nRep} {incr i} {
       set term [list [random_term] [random_term]]
       set match "\"$term\""
-      do_test fts3rnd-1.$nodesize.$iTest.3.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.3.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+        SELECT docid FROM t1 WHERE t1 MATCH $match
       } [simple_phrase $term]
     }
 
     # Three word phrases.
     #
-    for {set i 0} {$i < 10} {incr i} {
+    for {set i 0} {$i < $nRep} {incr i} {
       set term [list [random_term] [random_term] [random_term]]
       set match "\"$term\""
-      do_test fts3rnd-1.$nodesize.$iTest.4.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.4.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+        SELECT docid FROM t1 WHERE t1 MATCH $match
       } [simple_phrase $term]
     }
 
     # Three word phrases made up of term-prefixes.
     #
-    for {set i 0} {$i < 10} {incr i} {
+    for {set i 0} {$i < $nRep} {incr i} {
       set    query "[string range [random_term] 0 1]* "
       append query "[string range [random_term] 0 1]* "
       append query "[string range [random_term] 0 1]*"
 
       set match "\"$query\""
-      do_test fts3rnd-1.$nodesize.$iTest.5.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+        SELECT docid FROM t1 WHERE t1 MATCH $match
       } [simple_phrase $query]
     }
 
     # A NEAR query with terms as the arguments.
     #
-    for {set i 0} {$i < 10} {incr i} {
+    for {set i 0} {$i < $nRep} {incr i} {
       set terms [list [random_term] [random_term]]
       set match [join $terms " NEAR "]
-      do_test fts3rnd-1.$nodesize.$iTest.6.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.6.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+        SELECT docid FROM t1 WHERE t1 MATCH $match 
       } [simple_near $terms 10]
     }
 
     # A 3-way NEAR query with terms as the arguments.
     #
-    for {set i 0} {$i < 10} {incr i} {
+    for {set i 0} {$i < $nRep} {incr i} {
       set terms [list [random_term] [random_term] [random_term]]
       set nNear 11
       set match [join $terms " NEAR/$nNear "]
       set fts3 [execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }]
-      do_test fts3rnd-1.$nodesize.$iTest.7.$i {
+      do_select_test fts3rnd-1.$nodesize.$iTest.7.$i {
-        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+        SELECT docid FROM t1 WHERE t1 MATCH $match
       } [simple_near $terms $nNear]
     }
     
@@ -255,12 +262,12 @@ foreach nodesize {50 500 1000 2000} {
       9  NOT setop_not
       10 AND setop_and
     } {
-      for {set i 0} {$i < 10} {incr i} {
+      for {set i 0} {$i < $nRep} {incr i} {
         set term1 [random_term]
         set term2 [random_term]
         set match "$term1 $op $term2"
-        do_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
+        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
-          execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+          SELECT docid FROM t1 WHERE t1 MATCH $match
         } [$proc [simple_phrase $term1] [simple_phrase $term2]]
       }
     }
@@ -272,14 +279,14 @@ foreach nodesize {50 500 1000 2000} {
       9  NOT setop_not
       10 AND setop_and
     } {
-      for {set i 0} {$i < 10} {incr i} {
+      for {set i 0} {$i < $nRep} {incr i} {
         set term1 [random_term]
         set term2 [random_term]
         set term3 [random_term]
         set term4 [random_term]
         set match "$term1 NEAR $term2 $op $term3 NEAR $term4"
-        do_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
+        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
-          execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
+          SELECT docid FROM t1 WHERE t1 MATCH $match
         } [$proc                                  \
             [simple_near [list $term1 $term2] 10] \
             [simple_near [list $term3 $term4] 10]