Fix a problem allowing a COMMIT following an OOM to cause fts3/4 corruption.

FossilOrigin-Name: 7f41d7006db4225cf9b3d197d3a76842778669ac079e76361214a8023c9976e6

ext/fts3/fts3.c

@@ -3890,6 +3890,8 @@ static int fts3RenameMethod(
     rc = sqlite3Fts3PendingTermsFlush(p);
   }
 
+  p->bIgnoreSavepoint = 1;
+
   if( p->zContentTbl==0 ){
     fts3DbExec(&rc, db,
       "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
@@ -3917,6 +3919,8 @@ static int fts3RenameMethod(
     "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
     p->zDb, p->zName, zName
   );
+
+  p->bIgnoreSavepoint = 0;
   return rc;
 }
 
@@ -3927,12 +3931,28 @@ static int fts3RenameMethod(
 */
 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   int rc = SQLITE_OK;
-  UNUSED_PARAMETER(iSavepoint);
+  Fts3Table *pTab = (Fts3Table*)pVtab;
-  assert( ((Fts3Table *)pVtab)->inTransaction );
+  assert( pTab->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
+  assert( pTab->mxSavepoint<=iSavepoint );
-  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
+  TESTONLY( pTab->mxSavepoint = iSavepoint );
-  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
+
-    rc = fts3SyncMethod(pVtab);
+  if( pTab->bIgnoreSavepoint==0 ){
+    if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){
+      char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
+          pTab->zDb, pTab->zName, pTab->zName
+          );
+      if( zSql ){
+        pTab->bIgnoreSavepoint = 1;
+        rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
+        pTab->bIgnoreSavepoint = 0;
+        sqlite3_free(zSql);
+      }else{
+        rc = SQLITE_NOMEM;
+      }
+    }
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint+1;
+    }
   }
   return rc;
 }
@@ -3943,12 +3963,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+  Fts3Table *pTab = (Fts3Table*)pVtab;
-  UNUSED_PARAMETER(iSavepoint);
+  assert( pTab->inTransaction );
-  UNUSED_PARAMETER(pVtab);
+  assert( pTab->mxSavepoint >= iSavepoint );
-  assert( p->inTransaction );
+  TESTONLY( pTab->mxSavepoint = iSavepoint-1 );
-  assert( p->mxSavepoint >= iSavepoint );
+  pTab->iSavepoint = iSavepoint;
-  TESTONLY( p->mxSavepoint = iSavepoint-1 );
   return SQLITE_OK;
 }
 
@@ -3958,11 +3977,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** Discard the contents of the pending terms table.
 */
 static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts3Table *p = (Fts3Table*)pVtab;
+  Fts3Table *pTab = (Fts3Table*)pVtab;
   UNUSED_PARAMETER(iSavepoint);
-  assert( p->inTransaction );
+  assert( pTab->inTransaction );
-  TESTONLY( p->mxSavepoint = iSavepoint );
+  TESTONLY( pTab->mxSavepoint = iSavepoint );
-  sqlite3Fts3PendingTermsClear(p);
+  if( (iSavepoint+1)<=pTab->iSavepoint ){
+    sqlite3Fts3PendingTermsClear(pTab);
+  }
   return SQLITE_OK;
 }
 

ext/fts3/fts3Int.h

@@ -265,6 +265,7 @@ struct Fts3Table {
   int nPgsz;                      /* Page size for host database */
   char *zSegmentsTbl;             /* Name of %_segments table */
   sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
+  int iSavepoint;
 
   /* 
   ** The following array of hash tables is used to buffer pending index 

ext/fts3/fts3_write.c

@@ -3325,7 +3325,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
     rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
     if( rc==SQLITE_DONE ) rc = SQLITE_OK;
   }
-  sqlite3Fts3PendingTermsClear(p);
 
   /* Determine the auto-incr-merge setting if unknown.  If enabled,
   ** estimate the number of leaf blocks of content to be written
@@ -3347,6 +3346,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
       rc = sqlite3_reset(pStmt);
     }
   }
+
+  if( rc==SQLITE_OK ){
+    sqlite3Fts3PendingTermsClear(p);
+  }
   return rc;
 }
 
@@ -5437,8 +5440,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     rc = fts3DoIncrmerge(p, &zVal[6]);
   }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
     rc = fts3DoAutoincrmerge(p, &zVal[10]);
+  }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){
+    rc = sqlite3Fts3PendingTermsFlush(p);
+  }
 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-  }else{
+  else{
     int v;
     if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
       v = atoi(&zVal[9]);
@@ -5456,8 +5462,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
       if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
       rc = SQLITE_OK;
     }
-#endif
   }
+#endif
   return rc;
 }
 

ext/fts5/fts5_main.c

@@ -2635,7 +2635,6 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   int rc = SQLITE_OK;
   char *zSql = 0;
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
 
   if( pTab->bInSavepoint==0 ){
@@ -2666,7 +2665,6 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   int rc = SQLITE_OK;
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
   if( (iSavepoint+1)<pTab->iSavepoint ){
     rc = sqlite3Fts5FlushToDisk(&pTab->p);
@@ -2685,7 +2683,6 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   int rc = SQLITE_OK;
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
   fts5TripCursors(pTab);
   pTab->p.pConfig->pgsz = 0;

ext/fts5/test/fts5faultG.test

@@ -30,7 +30,7 @@ do_execsql_test 1.0 {
   INSERT INTO t1 VALUES(' actually other stuff instead');
 }
 faultsim_save_and_close
-do_faultsim_test 1 -faults oom-t* -prep { 
+do_faultsim_test 1 -faults oom* -prep { 
   faultsim_restore_and_reopen
   execsql {
     BEGIN;

ext/fts5/test/fts5savepoint.test

@@ -71,7 +71,7 @@ ifcapable fts3 {
 
   do_catchsql_test 3.2 {
     DROP TABLE vt1;
-  } {1 {SQL logic error}}
+  } {0 {}}
 
   do_execsql_test 3.3 {
     SAVEPOINT x;

manifest

@@ -1,5 +1,5 @@
-C Add\sthe\sSQLITE_TESTCTRL_FK_NO_ACTION\stest\scontrol.\s\sMake\sit\savailable\sin\sthe\nCLI.\s\sFix\sa\sminor\sproblems\swith\sSQLITE_CHANGESETAPPLY_FKNOACTION\sin\ssessions.
+C Fix\sa\sproblem\sallowing\sa\sCOMMIT\sfollowing\san\sOOM\sto\scause\sfts3/4\scorruption.
-D 2023-10-21T16:33:20.752
+D 2023-10-21T18:12:07.638
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -60,9 +60,9 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 89298f42a071d4f08588a308cf2904fc5014f59b47ea48949a127b5747e9b90a
+F ext/fts3/fts3.c c409b5f9211dbe9336210435ef3bc936e54c4f2ad9b92c9a7cd5442cbbbf1411
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h e573c6d881f7238d77cc3fd2396cbb9b2fe13efef7d2ad295a155151c4e7efbd
+F ext/fts3/fts3Int.h be688580701d41340de73384e3acc8c55be12a438583207444bd5e20f9ef426c
 F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3
 F ext/fts3/fts3_expr.c 903bfb9433109fffb10e910d7066c49cbf8eeae316adc93f0499c4da7dfc932a
 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
@@ -78,7 +78,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154
 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226
 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
-F ext/fts3/fts3_write.c 3a0043bb527c5d11e90493fd9fead30836e161a5330ba2ba739b372b03ea0459
+F ext/fts3/fts3_write.c 16df5ea9ff3634821003ac71a0275e3af030517b8768680062f425a347aab5f0
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674
@@ -95,7 +95,7 @@ F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d
 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
 F ext/fts5/fts5_index.c 730c9c32ada18ce1eb7ff847b36507f4b005d88d47af7b47db521e695a8ea4c7
-F ext/fts5/fts5_main.c e345282be41c646702bbb91d488b78c35d9fa6e5a375d2809d9eb7dfbaed3314
+F ext/fts5/fts5_main.c 1ce6c8f446afbaaf22f4e1ccc0ec46d653168545b89a53bdbba0beddda820bec
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -167,7 +167,7 @@ F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05c
 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996
 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
-F ext/fts5/test/fts5faultG.test 7528654556b047ac515fc2790c03f71475141d5522a28edd3a6a630af269d9b1
+F ext/fts5/test/fts5faultG.test 340e59d2c2c1c7c379224f3968ee8d09b0f64bf56c5194217d1ded887b9d47c4
 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -199,7 +199,7 @@ F ext/fts5/test/fts5rank.test 30f29e278cd7fb8831ba4f082feb74d8eb90c463bf07113ae2
 F ext/fts5/test/fts5rebuild.test 55d6f17715cddbf825680dd6551efbc72ed916d8cf1cde40a46fc5d785b451e7
 F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415
 F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff171013003d8297203a20ad6
-F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e332d654375690f8e6e43f
+F ext/fts5/test/fts5savepoint.test 050796b24929325cdbbb2fbfe2794816ae95d298e940ae15032200c2f4a73725
 F ext/fts5/test/fts5secure.test a02f771742fb2b1b9bdcb4bf523bcf2d0aa1ff597831d40fe3e72aaa6d0ec40f
 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc
 F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
@@ -1130,7 +1130,7 @@ F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d
 F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57
 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0
 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f
-F test/fts3corrupt4.test 64c6729721575e16bb7fdfc3d23092bb84ac032b0980b891a10483cb84bc1931
+F test/fts3corrupt4.test 48bd57baed9654e511709a02dbef2d22ee54c012ad466e8648f0f825233faa08
 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5
 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525
 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf
@@ -2135,9 +2135,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9deb8b6915e794bf9c5de88f2727ef9857351eaf905f8ab56754bc4d399c88b4 a50a333ae11ba5d92f432108308ac0bec9afb00f466b78c8d3f3aa7e2851ef21
+P 91b64c6a70744704fc3285be7d8d46ba679ea5f0f69bd77b0e093eeb7447947d
-R 04a86bb9860fea6216c599ef4bafe3b4
+R 40550ff3002f44e1c120a49acdcaa100
-T +closed a50a333ae11ba5d92f432108308ac0bec9afb00f466b78c8d3f3aa7e2851ef21
+U dan
-U drh
+Z cc7f79151d80761539bb72edb302f3f6
-Z b0ccf6a50f82a49d4b230a7f548e7633
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-91b64c6a70744704fc3285be7d8d46ba679ea5f0f69bd77b0e093eeb7447947d
+7f41d7006db4225cf9b3d197d3a76842778669ac079e76361214a8023c9976e6

test/fts3corrupt4.test

@@ -2595,17 +2595,13 @@ do_execsql_test 17.1 {
     UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thread*';
 }
 
-do_catchsql_test 17.2 {
+do_execsql_test 17.2 {
-  DROP TABLE IF EXISTS t1;
-} {1 {SQL logic error}}
-
-do_execsql_test 17.3 {
   INSERT INTO t1(t1) VALUES('optimize');
 }
 
-do_catchsql_test 17.4 {
+do_catchsql_test 17.3 {
   DROP TABLE IF EXISTS t1;
-} {1 {SQL logic error}}
+} {0 {}}
 
 #-------------------------------------------------------------------------
 reset_db