Improve test coverage of fts5_index.c.

FossilOrigin-Name: c1f76686cee3918b1be785a4071d68cb3afda0ef
This commit is contained in:
dan 2015-09-14 19:51:05 +00:00
parent b79d5529af
commit dafeda754b
6 changed files with 313 additions and 43 deletions

View File

@ -1427,7 +1427,6 @@ static void fts5SegIterNextPage(
fts5DataRelease(pIter->pLeaf);
pIter->iLeafPgno++;
if( pIter->pNextLeaf ){
assert( pIter->iLeafPgno<=pSeg->pgnoLast );
pIter->pLeaf = pIter->pNextLeaf;
pIter->pNextLeaf = 0;
}else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
@ -1659,10 +1658,8 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
));
if( pNew ){
if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
if( pIter->iTermLeafOffset<pNew->szLeaf ){
pIter->pLeaf = pNew;
pIter->iLeafOffset = pIter->iTermLeafOffset;
}
pIter->pLeaf = pNew;
pIter->iLeafOffset = pIter->iTermLeafOffset;
}else{
int iRowidOff;
iRowidOff = fts5LeafFirstRowidOff(pNew);
@ -3168,7 +3165,7 @@ static void fts5WriteAppendTerm(
Fts5PageWriter *pPage = &pWriter->writer;
Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
if( p->rc ) return;
assert( p->rc==SQLITE_OK );
assert( pPage->buf.n>=4 );
assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );
@ -3315,16 +3312,12 @@ static void fts5WriteFinish(
int i;
Fts5PageWriter *pLeaf = &pWriter->writer;
if( p->rc==SQLITE_OK ){
if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){
*pnLeaf = 0;
}else{
if( pLeaf->buf.n>4 ){
fts5WriteFlushLeaf(p, pWriter);
}
*pnLeaf = pLeaf->pgno-1;
fts5WriteFlushBtree(p, pWriter);
assert( pLeaf->pgno>=1 );
if( pLeaf->buf.n>4 ){
fts5WriteFlushLeaf(p, pWriter);
}
*pnLeaf = pLeaf->pgno-1;
fts5WriteFlushBtree(p, pWriter);
}
fts5BufferFree(&pLeaf->term);
fts5BufferFree(&pLeaf->buf);
@ -3763,9 +3756,8 @@ static void fts5FlushOneHash(Fts5Index *p){
sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm);
if( writer.bFirstRowidInPage==0
&& pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1)
){
assert( writer.bFirstRowidInPage==0 );
if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
/* The entire doclist will fit on the current leaf. */
fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
}else{
@ -3773,8 +3765,6 @@ static void fts5FlushOneHash(Fts5Index *p){
i64 iDelta = 0;
int iOff = 0;
/* writer.bFirstRowidInPage = 0; */
/* The entire doclist will not fit on this leaf. The following
** loop iterates through the poslists that make up the current
** doclist. */
@ -4155,7 +4145,7 @@ static void fts5SetupPrefixIter(
i64 iRowid = fts5MultiIterRowid(p1);
int nTerm;
const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
if( doclist.n>0 && iRowid<=iLastRowid ){
@ -5290,7 +5280,6 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
if( iOff<n ){
i64 iDelta;
iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);
if( iDelta==0 ) return iOff;
iDocid += iDelta;
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
}

View File

@ -24,18 +24,23 @@ ifcapable !fts5 {
}
sqlite3_fts5_may_be_corrupt 1
proc create_t1 {} {
expr srand(0)
db func rnddoc fts5_rnddoc
db eval {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
}
if 1 {
# Create a simple FTS5 table containing 100 documents. Each document
# contains 10 terms, each of which start with the character "x".
#
expr srand(0)
db func rnddoc fts5_rnddoc
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
set mask [expr 31 << 31]
do_test 1.0 { create_t1 } {}
do_test 1.1 {
# Pick out the rowid of the right-most b-tree leaf in the new segment.
@ -75,6 +80,179 @@ do_execsql_test 2.2 {
SELECT length(block) FROM t2_data WHERE id=1;
} {2}
#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}
do_execsql_test 3.1 {
SELECT count(*) FROM t1_data;
} {105}
proc do_3_test {tn} {
set i 0
foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] {
incr i
do_test $tn.$i {
db eval BEGIN
db eval {DELETE FROM t1_data WHERE rowid = $::rowid}
list [
catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg
] $msg
} {1 {database disk image is malformed}}
catch { db eval ROLLBACK }
}
}
do_3_test 3.2
do_execsql_test 3.3 {
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 SELECT x FROM t1;
INSERT INTO t1(t1) VALUES('optimize');
} {}
do_3_test 3.4
do_test 3.5 {
execsql {
DELETE FROM t1;
INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
}
for {set i 0} {$i < 1000} {incr i} {
set rnd [expr int(rand() * 1000)]
set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]]
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
} {}
do_3_test 3.6
do_test 3.7 {
execsql {
INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
INSERT INTO t1 SELECT x FROM t1;
INSERT INTO t1(t1) VALUES('optimize');
}
} {}
do_3_test 3.8
do_test 3.9 {
execsql {
DELETE FROM t1;
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
for {set i 0} {$i < 100} {incr i} {
set rnd [expr int(rand() * 100)]
set doc "x[string repeat $rnd 20]"
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
} {}
do_3_test 3.10
}
#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
do_test 4.0 {
execsql {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
for {set i 0} {$i < 100} {incr i} {
set rnd [expr int(rand() * 100)]
set doc "x[string repeat $rnd 20]"
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
execsql { INSERT INTO t1(t1) VALUES('optimize') }
} {}
set nErr 0
for {set i 1} {1} {incr i} {
set struct [db one {SELECT block FROM t1_data WHERE id=10}]
binary scan $struct c* var
set end [lindex $var end]
if {$end<=$i} break
lset var end [expr $end - $i]
set struct [binary format c* $var]
db eval {
BEGIN;
UPDATE t1_data SET block = $struct WHERE id=10;
}
do_test 4.1.$i {
incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
set {} {}
} {}
catch { db eval ROLLBACK }
}
do_test 4.1.x { expr $nErr>45 } 1
#-------------------------------------------------------------------------
#
# The first argument passed to this command must be a binary blob
# containing an FTS5 leaf page. This command returns a copy of this
# blob, with the pgidx of the leaf page replaced by a single varint
# containing value $iVal.
#
proc rewrite_pgidx {blob iVal} {
binary scan $blob SS off1 szLeaf
if {$iVal<0 || $iVal>=128} {
error "$iVal out of range!"
} else {
set pgidx [binary format c $iVal]
}
binary format a${szLeaf}a* $blob $pgidx
}
reset_db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 40);
BEGIN;
INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj');
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1(x1) VALUES('optimize');
COMMIT;
}
#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b }
#
db func rewrite_pgidx rewrite_pgidx
set i 0
foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
foreach val {2 100} {
do_test 5.2.$val.[incr i] {
catchsql {
BEGIN;
UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid;
SELECT rowid FROM x1 WHERE x1 MATCH 'xa*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xb*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xc*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xd*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xe*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xf*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xg*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xh*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xi*';
}
set {} {}
} {}
catch { db eval ROLLBACK }
}
}
sqlite3_fts5_may_be_corrupt 0
finish_test

View File

@ -26,6 +26,8 @@ if { $tcl_platform(wordSize)<8 } {
return
}
if 1 {
proc do_fb_test {tn sql res} {
set res2 [lsort -integer -decr $res]
uplevel [list do_execsql_test $tn.1 $sql $res]
@ -128,5 +130,60 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
}
#--------------------------------------------------------------------
#
reset_db
set ::vocab [list \
IteratorpItercurrentlypointstothefirstrowidofadoclist \
Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
pageIfthecurrenttermisthelasttermonthepageloadthe \
doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
IteratorpItercurrentlypointstothefirstrowidofadoclist \
Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
pageIfthecurrenttermisthelasttermonthepageloadthe \
doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
]
proc rnddoc {} {
global vocab
set nVocab [llength $vocab]
set ret [list]
for {set i 0} {$i < 64} {incr i} {
lappend ret [lindex $vocab [expr $i % $nVocab]]
}
set ret
}
db func rnddoc rnddoc
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE abc USING fts5(a);
INSERT INTO abc(abc, rank) VALUES('pgsz', 32);
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc SELECT rnddoc() FROM abc;
INSERT INTO abc SELECT rnddoc() FROM abc;
}
do_execsql_test 3.2 {
INSERT INTO abc(abc) VALUES('integrity-check');
INSERT INTO abc(abc) VALUES('optimize');
INSERT INTO abc(abc) VALUES('integrity-check');
}
set v [lindex $vocab 0]
set i 0
foreach v $vocab {
do_execsql_test 3.3.[incr i] {
SELECT rowid FROM abc WHERE abc MATCH $v
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}
finish_test

View File

@ -14,7 +14,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault2
set testprefix fts5fault7
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
@ -22,6 +22,8 @@ ifcapable !fts5 {
return
}
if 1 {
#-------------------------------------------------------------------------
# Test fault-injection on a query that uses xColumnSize() on columnsize=0
# table.
@ -41,5 +43,49 @@ do_faultsim_test 1 -faults oom* -body {
faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
}
}
#-------------------------------------------------------------------------
# Test fault-injection when a segment is promoted.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a);
INSERT INTO t2(t2, rank) VALUES('automerge', 0);
INSERT INTO t2(t2, rank) VALUES('crisismerge', 4);
INSERT INTO t2(t2, rank) VALUES('pgsz', 40);
INSERT INTO t2 VALUES('a b c');
INSERT INTO t2 VALUES('d e f');
INSERT INTO t2 VALUES('f e d');
INSERT INTO t2 VALUES('c b a');
INSERT INTO t2 VALUES('a b c');
INSERT INTO t2 VALUES('d e f');
INSERT INTO t2 VALUES('f e d');
INSERT INTO t2 VALUES('c b a');
} {}
faultsim_save_and_close
do_faultsim_test 1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval {
BEGIN;
INSERT INTO t2 VALUES('c d c g g f');
INSERT INTO t2 VALUES('c d g b f d');
INSERT INTO t2 VALUES('c c f d e d');
INSERT INTO t2 VALUES('e a f c e f');
INSERT INTO t2 VALUES('c g f b b d');
INSERT INTO t2 VALUES('d a g a b b');
INSERT INTO t2 VALUES('e f a b c e');
INSERT INTO t2 VALUES('e c a g c d');
INSERT INTO t2 VALUES('g b d d e b');
INSERT INTO t2 VALUES('e a d a e d');
}
} -body {
db eval COMMIT
} -test {
faultsim_test_result {0 {}}
}
finish_test

View File

@ -1,5 +1,5 @@
C Testability\simprovements\sfor\sthe\sONEPASS_MULTI\senhancement.
D 2015-09-14T19:26:37.988
C Improve\stest\scoverage\sof\sfts5_index.c.
D 2015-09-14T19:51:05.852
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93
F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
F ext/fts5/fts5_index.c 62a682a70ea2e84fa67c7786ead892b201116ad1
F ext/fts5/fts5_index.c fae675f4d60cfff93b31f85576fddb6ade7d344e
F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c
F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059
F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@ -146,8 +146,8 @@ F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b
F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3
F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1
F ext/fts5/test/fts5corrupt3.test 618a965b4fd2859f9ddb72413c9828a23828704b
F ext/fts5/test/fts5dlidx.test 77259f6d8d671b486113b157bd30107ca9d6b0f6
F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43
@ -157,7 +157,7 @@ F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7
F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423
F ext/fts5/test/fts5fault7.test f4a9b796f8b20c78ec7cf9f4e11144d15d7c3fd4
F ext/fts5/test/fts5fault7.test f62ed4d98f137eb03f1db94d1fa41b17a771d971
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6
F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145
@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 379455af9fdfb299a857d46f35f0a76ed6daa08a
R 240d53905959924a39a1969f0c2c4fad
U drh
Z db76ee6590838fdbed88f2282d6173bf
P d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
R a767de21459f1cfc9798aa9a41d99081
U dan
Z 579f9e68a86182329093b3386940c228

View File

@ -1 +1 @@
d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
c1f76686cee3918b1be785a4071d68cb3afda0ef