mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add tests for, and source code comments to, fts5. No changes to code.

Browse Source 
FossilOrigin-Name: b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a
This commit is contained in:
dan 2023-09-04 16:48:31 +00:00
parent 25a6e6ec8b
commit dd24b1c8a1
5 changed files with 254 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
ext/fts5/fts5_index.c
View File

@ -5048,7 +5048,6 @@ static void fts5DoSecureDelete(
int iPgIdx = pSeg->pLeaf->szLeaf;
u64 iDelta = 0;
u64 iNextDelta = 0;
int iNextOff = 0;
int iOff = 0;
int nIdx = 0;
@ -5056,8 +5055,6 @@ static void fts5DoSecureDelete(
int bLastInDoclist = 0;
int iIdx = 0;
int iStart = 0;
int iKeyOff = 0;
int iPrevKeyOff = 0;
int iDelKeyOff = 0; /* Offset of deleted key, if any */
nIdx = nPg-iPgIdx;
@ -5082,10 +5079,21 @@ static void fts5DoSecureDelete(
** This block sets the following variables:
**
** iStart:
** The offset of the first byte of the rowid or delta-rowid
** value for the doclist entry being removed.
**
** iDelta:
** The value of the rowid or delta-rowid value for the doclist
** entry being removed.
**
** iNextOff:
** The offset of the next entry following the position list
** for the one being removed. If the position list for this
** entry overflows onto the next leaf page, this value will be
** greater than pLeaf->szLeaf.
*/
{
int iSOP;
int iSOP; /* Start-Of-Position-list */
if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
iStart = pSeg->iTermLeafOffset;
}else{
@ -5121,14 +5129,20 @@ static void fts5DoSecureDelete(
}
iOff = iStart;
/* Set variable bLastInDoclist to true if this entry happens to be
** the last rowid in the doclist for its term. */
if( iNextOff>=iPgIdx ){
int pgno = pSeg->iLeafPgno+1;
fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
iNextOff = iPgIdx;
}else{
/* Set bLastInDoclist to true if the entry being removed is the last
/* Loop through the page-footer. If iNextOff (offset of the
** entry following the one we are removing) is equal to the
** offset of a key on this page, then the entry is the last
** in its doclist. */
for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
int iKeyOff = 0;
for(iIdx=0; iIdx<nIdx; /* no-op */){
u32 iVal = 0;
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
iKeyOff += iVal;
@ -5138,30 +5152,47 @@ static void fts5DoSecureDelete(
}
}
if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
/* If this is (a) the first rowid on a page and (b) is not followed by
** another position list on the same page, set the "first-rowid" field
** of the header to 0. */
if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
fts5PutU16(&aPg[0], 0);
}
if( bLastInDoclist==0 ){
if( iNextOff!=iPgIdx ){
u64 iNextDelta = 0;
iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
}
}else if(
iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno
pSeg->iLeafPgno==pSeg->iTermLeafPgno
&& iStart==pSeg->iTermLeafOffset
){
/* The entry being removed was the only position list in its
** doclist. Therefore the term needs to be removed as well. */
int iKey = 0;
for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
int iKeyOff = 0;
/* Set iKeyOff to the offset of the term that will be removed - the
** last offset in the footer that is not greater than iStart. */
for(iIdx=0; iIdx<nIdx; iKey++){
u32 iVal = 0;
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
if( (iKeyOff+iVal)>(u32)iStart ) break;
iKeyOff += iVal;
}
assert_nc( iKey>=1 );
/* Set iDelKeyOff to the value of the footer entry to remove from
** the page. */
iDelKeyOff = iOff = iKeyOff;
if( iNextOff!=iPgIdx ){
/* This is the only position-list associated with the term, and there
** is another term following it on this page. So the subsequent term
** needs to be moved to replace the term associated with the entry
** being removed. */
int nPrefix = 0;
int nSuffix = 0;
int nPrefix2 = 0;
@ -5198,80 +5229,81 @@ static void fts5DoSecureDelete(
}
}
}else if( iStart==4 ){
int iPgno;
int iPgno;
assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
/* The entry being removed may be the only position list in
** its doclist. */
for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
int bEmpty = (pPg && pPg->nn==4);
fts5DataRelease(pPg);
if( bEmpty==0 ) break;
}
assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
/* The entry being removed may be the only position list in
** its doclist. */
for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
int bEmpty = (pPg && pPg->nn==4);
fts5DataRelease(pPg);
if( bEmpty==0 ) break;
}
if( iPgno==pSeg->iTermLeafPgno ){
i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
Fts5Data *pTerm = fts5DataRead(p, iId);
if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
int nTermIdx = pTerm->nn - pTerm->szLeaf;
int iTermIdx = 0;
int iTermOff = 0;
if( iPgno==pSeg->iTermLeafPgno ){
i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
Fts5Data *pTerm = fts5DataRead(p, iId);
if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
int nTermIdx = pTerm->nn - pTerm->szLeaf;
int iTermIdx = 0;
int iTermOff = 0;
while( 1 ){
u32 iVal = 0;
int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
iTermOff += iVal;
if( (iTermIdx+nByte)>=nTermIdx ) break;
iTermIdx += nByte;
}
nTermIdx = iTermIdx;
memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
fts5PutU16(&pTerm->p[2], iTermOff);
fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
if( nTermIdx==0 ){
fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
}
while( 1 ){
u32 iVal = 0;
int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
iTermOff += iVal;
if( (iTermIdx+nByte)>=nTermIdx ) break;
iTermIdx += nByte;
}
fts5DataRelease(pTerm);
nTermIdx = iTermIdx;
memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
fts5PutU16(&pTerm->p[2], iTermOff);
fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
if( nTermIdx==0 ){
fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
}
}
fts5DataRelease(pTerm);
}
}
if( p->rc==SQLITE_OK ){
const int nMove = nPg - iNextOff; /* Number of bytes to move */
int nShift = iNextOff - iOff; /* Distance to move them */
int iKeyOff = 0;
int iPrevKeyOff = 0;
memmove(&aPg[iOff], &aPg[iNextOff], nMove);
iPgIdx -= nShift;
nPg = iPgIdx;
fts5PutU16(&aPg[2], iPgIdx);
for(iIdx=0; iIdx<nIdx; /* no-op */){
u32 iVal = 0;
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
iKeyOff += iVal;
if( iKeyOff!=iDelKeyOff ){
if( iKeyOff>iOff ){
iKeyOff -= nShift;
nShift = 0;
}
nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
iPrevKeyOff = iKeyOff;
}
}
if( p->rc==SQLITE_OK ){
const int nMove = nPg - iNextOff;
int nShift = 0;
memmove(&aPg[iOff], &aPg[iNextOff], nMove);
iPgIdx -= (iNextOff - iOff);
nPg = iPgIdx;
fts5PutU16(&aPg[2], iPgIdx);
nShift = iNextOff - iOff;
for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
u32 iVal = 0;
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
iKeyOff += iVal;
if( iKeyOff!=iDelKeyOff ){
if( iKeyOff>iOff ){
iKeyOff -= nShift;
nShift = 0;
}
nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
iPrevKeyOff = iKeyOff;
}
}
if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
}
assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
}
sqlite3_free(aIdx);
assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg);
}
sqlite3_free(aIdx);
}
/*

21
ext/fts5/test/fts5secure6.test
View File

@ -70,5 +70,26 @@ do_execsql_test 2.2 {
SELECT rowid FROM t1('def')
} {-100000 -99999 9223372036854775800}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd)
}
do_execsql_test 2.1 {
BEGIN;
INSERT INTO t1(rowid, x)
VALUES(51869, 'when whenever where weress what turn'),
(51871, 'to were');
COMMIT;
}
do_execsql_test 3.2 {
DELETE FROM t1 WHERE rowid=51871;
INSERT INTO t1(t1) VALUES('integrity-check');
}
finish_test

116
ext/fts5/test/fts5secure7.test Normal file
View File

@ -0,0 +1,116 @@
# 2023 Feb 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# TESTRUNNER: slow
#
source [file join [file dirname [info script]] fts5_common.tcl]
ifcapable !fts5 { finish_test ; return }
set ::testprefix fts5secure7
set NVOCAB 500
set NDOC [expr 1000]
set NREP 100
set nDeletePerRep [expr 5]
set VOCAB [list]
proc select_one {list} {
set n [llength $list]
lindex $list [expr {abs(int(rand()*$n))}]
}
proc init_vocab {} {
set L [split "abcdefghijklmnopqrstuvwxyz" {}]
set nL [llength $L]
for {set i 0} {$i < $::NVOCAB} {incr i} {
set n [expr {6 + int(rand()*8)}]
set word ""
for {set j 0} {$j < $n} {incr j} {
append word [select_one $L]
}
lappend ::VOCAB $word
}
}
proc get_word {} {
select_one $::VOCAB
}
proc get_document {nWord} {
set ret [list]
for {set i 0} {$i < $nWord} {incr i} {
lappend ret [get_word]
}
return $ret
}
init_vocab
db func document [list get_document 12]
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(body);
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
}
do_execsql_test 1.1 {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
)
INSERT INTO t1 SELECT document() FROM s;
}
for {set iRep 0} {$iRep < $NREP} {incr iRep} {
set lRowid [db eval {SELECT rowid FROM t1}]
for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} {
set idx [select_one $lRowid]
db eval {
DELETE FROM t1 WHERE rowid=$idx
}
}
db eval {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep
)
INSERT INTO t1 SELECT document() FROM s;
}
do_execsql_test 1.2.$iRep {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
reset_db
db func document [list get_document 12]
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(body);
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
INSERT INTO t1(t1, rank) VALUES('pgsz', 128);
}
do_execsql_test 2.1 {
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
)
INSERT INTO t1 SELECT document() FROM s;
}
for {set ii 0} {$ii < $NDOC} {incr ii} {
set lRowid [db eval {SELECT rowid FROM t1}]
set idx [select_one $lRowid]
db eval { DELETE FROM t1 WHERE rowid=$idx }
do_execsql_test 2.2.$ii {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
finish_test

17
manifest
View File

@ -1,5 +1,5 @@
C Do\snot\smake\sassumptions\sabout\sthe\sbyteorder\sof\sPowerPC\sprocessors.
D 2023-09-04T12:50:17.388
C Add\stests\sfor,\sand\ssource\scode\scomments\sto,\sfts5.\sNo\schanges\sto\scode.
D 2023-09-04T16:48:31.747
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
F ext/fts5/fts5_index.c 77bd70d50cb8397f3d8465cc4894dcdac75aa5e1fb3bbc5a4a5bc15e12fced97
F ext/fts5/fts5_index.c 47c290589fa38f6a1860b4fc26716ed30d79ddc283b38813d1c8c1d702108ab8
F ext/fts5/fts5_main.c 7070031993ba5b5d89b13206ec4ef624895f2f7c0ec72725913d301e4d382445
F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@ -204,7 +204,8 @@ F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd
F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
F ext/fts5/test/fts5secure6.test a0a28cfb9bf9721408b65b5d7c7ce369af3d688e273da24d101c25d60cdce05c
F ext/fts5/test/fts5secure6.test 120feecc8c55b4774f858721e6c62c2094b059ecbcfd7fdc24bde886f55ef6ca
F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
@ -2115,8 +2116,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 2aef9af3dd493a82ba090ccb8ab1b8974349392ec3f1c35db7a3dbbec41b5c51
R a87cf75daed42b21bba72eb7f67c969b
U drh
Z 72c7bea784c6c2657adbd0596be177aa
P 4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61
R e68bce01327834d099d97be2fe3709dd
U dan
Z a7ec9ec1221680dfc975891082f06d30
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61
b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a