Fix various problems with fts5 contentless_delete=1 tables.
FossilOrigin-Name: 0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
This commit is contained in:
dan
parent
3f47e7551c
commit
d05bf0fe61
@ -3038,13 +3038,19 @@ static int fts5IndexTombstoneQuery(
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
** Return true if the iterator passed as the only argument points
|
||||
** to an segment entry for which there is a tombstone. Return false
|
||||
** if there is no tombstone or if the iterator is already at EOF.
|
||||
*/
|
||||
static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
|
||||
int iFirst = pIter->aFirst[1].iFirst;
|
||||
Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
|
||||
|
||||
if( pSeg->nTombstone ){
|
||||
if( pSeg->pLeaf && pSeg->nTombstone ){
|
||||
/* Figure out which page the rowid might be present on. */
|
||||
int iPg = pSeg->iRowid % pSeg->nTombstone;
|
||||
int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone;
|
||||
assert( iPg>=0 );
|
||||
|
||||
if( pSeg->apTombstone[iPg]==0 ){
|
||||
pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
|
||||
@ -6428,7 +6434,12 @@ int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
|
||||
/*
|
||||
** Buffer pPg contains a page of a tombstone hash table - one of nPg.
|
||||
*/
|
||||
static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
|
||||
static int fts5IndexTombstoneAddToPage(
|
||||
Fts5Data *pPg,
|
||||
int bForce,
|
||||
int nPg,
|
||||
u64 iRowid
|
||||
){
|
||||
int szKey = TOMBSTONE_KEYSIZE(pPg);
|
||||
int nSlot = (pPg->nn - 8) / szKey;
|
||||
int iSlot = (iRowid / nPg) % nSlot;
|
||||
@ -6440,6 +6451,10 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
|
||||
return 0;
|
||||
}
|
||||
|
||||
if( bForce==0 && nElem>=(nSlot/2) ){
|
||||
return 1;
|
||||
}
|
||||
|
||||
fts5PutU32(&pPg->p[4], nElem+1);
|
||||
if( szKey==4 ){
|
||||
u32 *aSlot = (u32*)&pPg->p[8];
|
||||
@ -6451,7 +6466,7 @@ static int fts5IndexTombstoneAddToPage(Fts5Data *pPg, int nPg, u64 iRowid){
|
||||
fts5PutU64((u8*)&aSlot[iSlot], iRowid);
|
||||
}
|
||||
|
||||
return nElem >= (nSlot/2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -6489,7 +6504,7 @@ static int fts5IndexTombstoneRehash(
|
||||
}
|
||||
|
||||
if( pData ){
|
||||
int szKeyIn = pData->p[3] ? 8 : 4;
|
||||
int szKeyIn = TOMBSTONE_KEYSIZE(pData);
|
||||
int nSlotIn = (pData->nn - 8) / szKeyIn;
|
||||
int iIn;
|
||||
for(iIn=0; iIn<nSlotIn; iIn++){
|
||||
@ -6507,7 +6522,7 @@ static int fts5IndexTombstoneRehash(
|
||||
/* If iVal is not 0 at this point, insert it into the new hash table */
|
||||
if( iVal ){
|
||||
Fts5Data *pPg = apOut[(iVal % nOut)];
|
||||
res = fts5IndexTombstoneAddToPage(pPg, nOut, iVal);
|
||||
res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal);
|
||||
if( res ) break;
|
||||
}
|
||||
}
|
||||
@ -6642,7 +6657,6 @@ static void fts5IndexTombstoneAdd(
|
||||
Fts5Data *pPg = 0;
|
||||
int iPg = -1;
|
||||
int szKey = 0;
|
||||
|
||||
int nHash = 0;
|
||||
Fts5Data **apHash = 0;
|
||||
|
||||
@ -6654,7 +6668,7 @@ static void fts5IndexTombstoneAdd(
|
||||
return;
|
||||
}
|
||||
|
||||
if( 0==fts5IndexTombstoneAddToPage(pPg, pSeg->nPgTombstone, iRowid) ){
|
||||
if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){
|
||||
fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn);
|
||||
fts5DataRelease(pPg);
|
||||
return;
|
||||
@ -6673,7 +6687,7 @@ static void fts5IndexTombstoneAdd(
|
||||
** table pages, then write them all out to disk. */
|
||||
if( nHash ){
|
||||
int ii = 0;
|
||||
fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], nHash, iRowid);
|
||||
fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid);
|
||||
for(ii=0; ii<nHash; ii++){
|
||||
i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii);
|
||||
fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
|
||||
@ -7544,7 +7558,26 @@ static void fts5DecodeFunction(
|
||||
}
|
||||
}else if( bTomb ){
|
||||
u32 nElem = fts5GetU32(&a[4]);
|
||||
int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8;
|
||||
int nSlot = (n - 8) / szKey;
|
||||
int ii;
|
||||
sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
|
||||
if( aBlob[1] ){
|
||||
sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0");
|
||||
}
|
||||
for(ii=0; ii<nSlot; ii++){
|
||||
u64 iVal = 0;
|
||||
if( szKey==4 ){
|
||||
u32 *aSlot = (u32*)&aBlob[8];
|
||||
if( aSlot[ii] ) iVal = fts5GetU32((u8*)&aSlot[ii]);
|
||||
}else{
|
||||
u64 *aSlot = (u64*)&aBlob[8];
|
||||
if( aSlot[ii] ) iVal = fts5GetU64((u8*)&aSlot[ii]);
|
||||
}
|
||||
if( iVal!=0 ){
|
||||
sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", (i64)iVal);
|
||||
}
|
||||
}
|
||||
}else if( iSegid==0 ){
|
||||
if( iRowid==FTS5_AVERAGES_ROWID ){
|
||||
fts5DecodeAverages(&rc, &s, a, n);
|
||||
|
||||
152
ext/fts5/test/fts5contentless2.test
Normal file
152
ext/fts5/test/fts5contentless2.test
Normal file
@ -0,0 +1,152 @@
|
||||
# 2023 July 19
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#***********************************************************************
|
||||
#
|
||||
# This file contains tests for the content= and content_rowid= options.
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
set testprefix fts5contentless2
|
||||
|
||||
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
|
||||
ifcapable !fts5 {
|
||||
finish_test
|
||||
return
|
||||
}
|
||||
|
||||
proc vocab {} {
|
||||
list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp
|
||||
}
|
||||
|
||||
proc document {nToken} {
|
||||
set doc [list]
|
||||
set vocab [vocab]
|
||||
for {set ii 0} {$ii < $nToken} {incr ii} {
|
||||
lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]]
|
||||
}
|
||||
set doc
|
||||
}
|
||||
db func document document
|
||||
|
||||
proc contains {doc token} {
|
||||
expr {[lsearch $doc $token]>=0}
|
||||
}
|
||||
db func contains contains
|
||||
|
||||
proc do_compare_tables_test {tn} {
|
||||
uplevel [list do_test $tn {
|
||||
foreach v [vocab] {
|
||||
set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }]
|
||||
set l2 [execsql { SELECT rowid FROM t2($v) }]
|
||||
if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
|
||||
|
||||
set w "[string range $v 0 1]*"
|
||||
set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
|
||||
set l2 [execsql { SELECT rowid FROM t2($w) }]
|
||||
if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
|
||||
|
||||
set w "[string range $v 0 0]*"
|
||||
set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
|
||||
set l2 [execsql { SELECT rowid FROM t2($w) }]
|
||||
if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
|
||||
|
||||
set l1 [execsql {
|
||||
SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC
|
||||
}]
|
||||
set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }]
|
||||
if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
|
||||
}
|
||||
set {} {}
|
||||
} {}]
|
||||
}
|
||||
|
||||
proc lshuffle {in} {
|
||||
set L [list]
|
||||
set ret [list]
|
||||
foreach elem $in { lappend L [list [expr rand()] $elem] }
|
||||
foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] }
|
||||
set ret
|
||||
}
|
||||
|
||||
expr srand(0)
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE t2 USING fts5(
|
||||
doc, prefix=2, content=, contentless_delete=1
|
||||
);
|
||||
|
||||
CREATE TABLE t1(doc);
|
||||
CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN
|
||||
DELETE FROM t2 WHERE rowid = old.rowid;
|
||||
END;
|
||||
}
|
||||
|
||||
set SMALLEST64 -9223372036854775808
|
||||
set LARGEST64 9223372036854775807
|
||||
|
||||
foreach {tn r1 r2} {
|
||||
1 0 50
|
||||
2 $SMALLEST64 $SMALLEST64+50
|
||||
3 $LARGEST64-50 $LARGEST64
|
||||
4 -50 -1
|
||||
} {
|
||||
set r1 [expr $r1]
|
||||
set r2 [expr $r2]
|
||||
|
||||
do_test 1.1.$tn {
|
||||
execsql BEGIN
|
||||
for {set ii $r1} {$ii <= $r2} {incr ii} {
|
||||
execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); }
|
||||
}
|
||||
execsql COMMIT
|
||||
} {}
|
||||
}
|
||||
do_test 1.2 {
|
||||
db eval { SELECT rowid, doc FROM t1 } {
|
||||
execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) }
|
||||
}
|
||||
} {}
|
||||
|
||||
foreach {tn rowid} {
|
||||
1 $SMALLEST64
|
||||
2 0
|
||||
3 -5
|
||||
4 -30
|
||||
5 $LARGEST64
|
||||
6 $LARGEST64-1
|
||||
} {
|
||||
set rowid [expr $rowid]
|
||||
do_execsql_test 1.3.$tn.1 {
|
||||
DELETE FROM t1 WHERE rowid=$rowid
|
||||
}
|
||||
do_compare_tables_test 1.3.$tn.2
|
||||
}
|
||||
|
||||
set iTest 1
|
||||
foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] {
|
||||
if {($iTest % 50)==0} {
|
||||
execsql { INSERT INTO t2(t2) VALUES('optimize') }
|
||||
}
|
||||
if {($iTest % 5)==0} {
|
||||
execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) }
|
||||
}
|
||||
do_execsql_test 1.4.$iTest.1($r) {
|
||||
DELETE FROM t1 WHERE rowid=$r
|
||||
}
|
||||
do_compare_tables_test 1.4.$iTest.2
|
||||
incr iTest
|
||||
}
|
||||
|
||||
do_execsql_test 1.5 {
|
||||
SELECT * FROM t1
|
||||
} {}
|
||||
|
||||
finish_test
|
||||
|
||||
15
manifest
15
manifest
@ -1,5 +1,5 @@
|
||||
C Merge\sthe\sFTS5\smemory\sleak\sfix\sfrom\strunk.
|
||||
D 2023-07-18T17:43:47.072
|
||||
C Fix\svarious\sproblems\swith\sfts5\scontentless_delete=1\stables.
|
||||
D 2023-07-18T19:52:32.733
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||
@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
|
||||
F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
|
||||
F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
|
||||
F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
|
||||
F ext/fts5/fts5_index.c 78c234c47a4870dd8dad7289f457188b8b8c039ce6ecf0e1d8aea0d369857522
|
||||
F ext/fts5/fts5_index.c a2e081dcffed12da4ec2a03429da368d8d0a980ddbed1324cd00f028fb510c48
|
||||
F ext/fts5/fts5_main.c ede405f0f11db562653b988d043a531daa66093b46c1b35b8fcddb54819cba84
|
||||
F ext/fts5/fts5_storage.c 7d22c8ea1d484134bd715f55b370ae9b5a830b627986344c4ffa532c3e89186b
|
||||
F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
|
||||
@ -133,6 +133,7 @@ F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac
|
||||
F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
|
||||
F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
|
||||
F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
|
||||
F ext/fts5/test/fts5contentless2.test 61ce8780ac933fb52bbba4191c33cc1f97bfa96eb7c1a7a2d6338c4bcfa6916a
|
||||
F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
|
||||
F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
|
||||
F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
|
||||
@ -2044,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 5f66eb4e2603278dcc9dbfe4bf506cba1aa03180cfb492a0dfc3a8be32cc994b 4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a
|
||||
R 6df4842f998a408f262f6f4830fd595f
|
||||
U drh
|
||||
Z 63584765e02f480fe2927dfa6f8c1662
|
||||
P fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
|
||||
R 0b35403fc3d2abc67db6df3c073d3f31
|
||||
U dan
|
||||
Z c153ae75657e3c69ee3dd162b0a421d3
|
||||
# Remove this line to create a well-formed Fossil manifest.
|
||||
|
||||
@ -1 +1 @@
|
||||
fb65cb73d7ea22a8b20dccfa3abdaaa809eee4fcee6fe4846bd2e598ceb49aa4
|
||||
0d005112b8aca9e9eca9d86d5fed9168f6a0218fd290b5489b9e7b05714610f4
|
||||
Loading…
Reference in New Issue
Block a user