Add defensive code to better handle corrupt records in fts5.

FossilOrigin-Name: a99c29b5efaf17e34506cda5ce017ec0112b3cb24bf8a2f1831f0c14a8aa8d30

ext/fts5/fts5_buffer.c

@@ -178,6 +178,7 @@ int sqlite3Fts5PoslistNext64(
     i64 iOff = *piOff;
     int iVal;
     fts5FastGetVarint32(a, i, iVal);
+    assert( iVal>=0 );
     if( iVal<=1 ){
       if( iVal==0 ){
         *pi = i;
@@ -191,9 +192,12 @@ int sqlite3Fts5PoslistNext64(
         *piOff = -1;
         return 1;
       }
+      *piOff = iOff + ((iVal-2) & 0x7FFFFFFF);
+    }else{
+      *piOff = (iOff & (i64)0x7FFFFFFF<<32)+((iOff + (iVal-2)) & 0x7FFFFFFF);
     }
-    *piOff = iOff + ((iVal-2) & 0x7FFFFFFF);
     *pi = i;
+    assert( *piOff>=iOff );
     return 0;
   }
 }
@@ -232,14 +236,16 @@ void sqlite3Fts5PoslistSafeAppend(
   i64 *piPrev, 
   i64 iPos
 ){
-  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
-  if( (iPos & colmask) != (*piPrev & colmask) ){
-    pBuf->p[pBuf->n++] = 1;
-    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
-    *piPrev = (iPos & colmask);
+  if( iPos>=*piPrev ){
+    static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
+    if( (iPos & colmask) != (*piPrev & colmask) ){
+      pBuf->p[pBuf->n++] = 1;
+      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
+      *piPrev = (iPos & colmask);
+    }
+    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
+    *piPrev = iPos;
   }
-  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
-  *piPrev = iPos;
 }
 
 int sqlite3Fts5PoslistWriterAppend(

ext/fts5/fts5_index.c

@@ -4979,7 +4979,7 @@ static void fts5MergePrefixLists(
   Fts5Buffer *aBuf                /* Other lists to merge in */ 
 ){
 #define fts5PrefixMergerNextPosition(p) \
-  sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos);
+  sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos)
 #define FTS5_MERGE_NLIST 16
   PrefixMerger aMerger[FTS5_MERGE_NLIST];
   PrefixMerger *pHead = 0;
@@ -5078,6 +5078,8 @@ static void fts5MergePrefixLists(
       nTail = pHead->iter.nPoslist - pHead->iOff;
 
       /* WRITEPOSLISTSIZE */
+      assert_nc( tmp.n+nTail<=nTmp );
+      assert( tmp.n+nTail<=nTmp+nMerge*10 );
       if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){
         if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
         break;

ext/fts5/test/fts5corrupt3.test

@@ -14791,7 +14791,7 @@ do_test 75.0 {
 
 do_catchsql_test 75.1 {
   SELECT rowid, quote(matchinfo(t1,'pcxybs')) FROM t1 WHERE t1 MATCH 'e*';
-} {1 {database disk image is malformed}}
+} {1 {unable to use function matchinfo in the requested context}}
 
 #-------------------------------------------------------------------------
 reset_db

manifest

@@ -1,5 +1,5 @@
-C Remove\stwo\sassert()s\sin\sFTS5\sthat\schecks\sfor\sdatabase\scorruption\sright\sbefore\nactual\scode\sdoes\sthe\ssame\scheck.\ndbsqlfuzz\s9680db8f5338cb9b6060eb3a02d984555d4472e3.
-D 2021-06-08T23:46:59.097
+C Add\sdefensive\scode\sto\sbetter\shandle\scorrupt\srecords\sin\sfts5.
+D 2021-06-09T13:39:11.202
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -115,11 +115,11 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
 F ext/fts5/fts5Int.h 26c74dd5776f798436fbf604a0bf0e8de263b35b5060b05c15f9085845d9fda2
 F ext/fts5/fts5_aux.c f558e1fb9f0f86a4f7489e258c162e1f947de5ff2709087fbb465fddb7092f98
-F ext/fts5/fts5_buffer.c 5a5fe0159752c0fb0a5a93c722e9db2662822709490769d482b76a6dc8aaca70
+F ext/fts5/fts5_buffer.c 605f509f88c73e9ab41323cb6cf74932f936acf3f7d9c87bd0d3c599b4c90281
 F ext/fts5/fts5_config.c 8336d0ff6db0933f63cfec8ae0ab76e68393259cbccc0b46e1f79f7fa1842ff3
 F ext/fts5/fts5_expr.c 9462249a3bb82d0e49b163500f9d2197c2e4cd95bf440a9bbfc3906b22ea1e1b
 F ext/fts5/fts5_hash.c 1aa93c9b5f461afba66701ee226297dc78402b3bdde81e90a10de5fe3df14959
-F ext/fts5/fts5_index.c 104ba1088ffbc1453ae8ae6f8ba896ab3ac62bb21829bf403b6c86eb7cd2857f
+F ext/fts5/fts5_index.c eb1864c6abacf08d959956183a55a4f9767af76be289f2bb519bb0f197b3fd72
 F ext/fts5/fts5_main.c 35ebbcae681a4a40027c47bc2e94d7e7c81e331dc406bb9b23c546454ee8f98a
 F ext/fts5/fts5_storage.c 58ba71e6cd3d43a5735815e7956ee167babb4d2cbfe206905174792af4d09d75
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -160,7 +160,7 @@ F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c0
 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
 F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
 F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
-F ext/fts5/test/fts5corrupt3.test a7c74304ade9934c51604f59d1f30e7e3f6309a340daaade1a13a857cc5921af
+F ext/fts5/test/fts5corrupt3.test 79467b7c69a9c45ee3260507c4648d6449323aaa3c1e05b360c9479f1aa7dcce
 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3
 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7
 F ext/fts5/test/fts5detail.test 31b240dbf6d44ac3507e2f8b65f29fdc12465ffd531212378c7ce1066766f54e
@@ -1918,7 +1918,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e3188a866362894d98521b3006fb5b9a63fe4a3bb2ff4f1b58acb6eb4e52da8f
-R 498a508e08d4a68fa12e51791716d97d
-U drh
-Z a25454f0647dd4b7e8930ab999ad558e
+P f3f82967b2e9fd4168d86300138ac4bbcd6ffd7a3ff5ecf20acdc29e5356ac3b
+R e887ea774358be61a01c8969bbba9b04
+U dan
+Z 47ac93f33c440d293182c1e4a3488493

manifest.uuid

@@ -1 +1 @@
-f3f82967b2e9fd4168d86300138ac4bbcd6ffd7a3ff5ecf20acdc29e5356ac3b
+a99c29b5efaf17e34506cda5ce017ec0112b3cb24bf8a2f1831f0c14a8aa8d30