diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ed919298e7..8804e94bb0 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3038,13 +3038,19 @@ static int fts5IndexTombstoneQuery( return 0; } +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; - if( pSeg->nTombstone ){ + if( pSeg->pLeaf && pSeg->nTombstone ){ /* Figure out which page the rowid might be present on. */ - int iPg = pSeg->iRowid % pSeg->nTombstone; + int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone; + assert( iPg>=0 ); if( pSeg->apTombstone[iPg]==0 ){ pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex, @@ -6428,7 +6434,12 @@ int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ /* ** Buffer pPg contains a page of a tombstone hash table - one of nPg. */ -static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ int szKey = TOMBSTONE_KEYSIZE(pPg); int nSlot = (pPg->nn - 8) / szKey; int iSlot = (iRowid / nPg) % nSlot; @@ -6440,6 +6451,10 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){ return 0; } + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + fts5PutU32(&pPg->p[4], nElem+1); if( szKey==4 ){ u32 *aSlot = (u32*)&pPg->p[8]; @@ -6451,7 +6466,7 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){ fts5PutU64((u8*)&aSlot[iSlot], iRowid); } - return nElem >= (nSlot/2); + return 0; } /* @@ -6489,7 +6504,7 @@ static int fts5IndexTombstoneRehash( } if( pData ){ - int szKeyIn = pData->p[3] ? 8 : 4; + int szKeyIn = TOMBSTONE_KEYSIZE(pData); int nSlotIn = (pData->nn - 8) / szKeyIn; int iIn; for(iIn=0; iInnPgTombstone, iRowid) ){ + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); fts5DataRelease(pPg); return; @@ -6673,7 +6687,7 @@ static void fts5IndexTombstoneAdd( ** table pages, then write them all out to disk. */ if( nHash ){ int ii = 0; - fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], nHash, iRowid); + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); for(ii=0; iiiSegid, ii); fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); @@ -7544,7 +7558,26 @@ static void fts5DecodeFunction( } }else if( bTomb ){ u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; ii=0} +} +db func contains contains + +proc do_compare_tables_test {tn} { + uplevel [list do_test $tn { + foreach v [vocab] { + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }] + set l2 [execsql { SELECT rowid FROM t2($v) }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 1]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 0]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set l1 [execsql { + SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC + }] + set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + } + set {} {} + } {}] +} + +proc lshuffle {in} { + set L [list] + set ret [list] + foreach elem $in { lappend L [list [expr rand()] $elem] } + foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] } + set ret +} + +expr srand(0) + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5( + doc, prefix=2, content=, contentless_delete=1 + ); + + CREATE TABLE t1(doc); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE rowid = old.rowid; + END; +} + +set SMALLEST64 -9223372036854775808 +set LARGEST64 9223372036854775807 + +foreach {tn r1 r2} { + 1 0 50 + 2 $SMALLEST64 $SMALLEST64+50 + 3 $LARGEST64-50 $LARGEST64 + 4 -50 -1 +} { + set r1 [expr $r1] + set r2 [expr $r2] + + do_test 1.1.$tn { + execsql BEGIN + for {set ii $r1} {$ii <= $r2} {incr ii} { + execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); } + } + execsql COMMIT + } {} +} +do_test 1.2 { + db eval { SELECT rowid, doc FROM t1 } { + execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) } + } +} {} + +foreach {tn rowid} { + 1 $SMALLEST64 + 2 0 + 3 -5 + 4 -30 + 5 $LARGEST64 + 6 $LARGEST64-1 +} { + set rowid [expr $rowid] + do_execsql_test 1.3.$tn.1 { + DELETE FROM t1 WHERE rowid=$rowid + } + do_compare_tables_test 1.3.$tn.2 +} + +set iTest 1 +foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] { + if {($iTest % 50)==0} { + execsql { INSERT INTO t2(t2) VALUES('optimize') } + } + if {($iTest % 5)==0} { + execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) } + } + do_execsql_test 1.4.$iTest.1($r) { + DELETE FROM t1 WHERE rowid=$r + } + do_compare_tables_test 1.4.$iTest.2 + incr iTest +} + +do_execsql_test 1.5 { + SELECT * FROM t1 +} {} + +finish_test + diff --git a/manifest b/manifest index b2e058d24d..53072385c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sFTS5\smemory\sleak\sfix\sfrom\strunk. -D 2023-07-18T17:43:47.072 +C Fix\svarious\sproblems\swith\sfts5\scontentless_delete=1\stables. +D 2023-07-18T19:52:32.733 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 78c234c47a4870dd8dad7289f457188b8b8c039ce6ecf0e1d8aea0d369857522 +F ext/fts5/fts5_index.c a2e081dcffed12da4ec2a03429da368d8d0a980ddbed1324cd00f028fb510c48 F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84 F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -133,6 +133,7 @@ F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283 F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9 +F ext/fts5/test/fts5contentless2.test 61ce8780ac933fb52bbba4191c33cc1f97bfa96eb7c1a7a2d6338c4bcfa6916a F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78 @@ -2044,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f66eb4e2603278dcc9dbfe4bf506cba1aa03180cfb492a0dfc3a8be32cc994b 4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a -R 6df4842f998a408f262f6f4830fd595f -U drh -Z 63584765e02f480fe2927dfa6f8c1662 +P fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4 +R 0b35403fc3d2abc67db6df3c073d3f31 +U dan +Z c153ae75657e3c69ee3dd162b0a421d3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 28852ffb55..aa3336730e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4 \ No newline at end of file +0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4 \ No newline at end of file