Add test cases. And some fixes.

FossilOrigin-Name: adee788586197445672013d434e7ba47ce510b59
This commit is contained in:
dan 2015-05-15 12:18:39 +00:00
parent 5d03b9d365
commit 415ae725cb
8 changed files with 213 additions and 68 deletions

View File

@ -2071,15 +2071,9 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
int pgnoLast = 0;
if( pDlidx ){
/* If the doclist-iterator is already at EOF, then the current doclist
** contains no entries except those on the current page. */
if( fts5DlidxIterEof(p, pDlidx)==0 ){
int iSegid = pIter->pSeg->iSegid;
pgnoLast = fts5DlidxIterPgno(pDlidx);
pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast));
}else{
pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel);
}
int iSegid = pIter->pSeg->iSegid;
pgnoLast = fts5DlidxIterPgno(pDlidx);
pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast));
}else{
int iOff; /* Byte offset within pLeaf */
Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
@ -3430,8 +3424,7 @@ static void fts5WriteInit(
pWriter->iSegid = iSegid;
pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter));
pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter));
if( pWriter->aDlidx==0 ) return;
if( fts5WriteDlidxGrow(p, pWriter, 1) ) return;
pWriter->nWriter = 1;
pWriter->nDlidx = 1;
pWriter->aWriter[0].pgno = 1;
@ -3902,7 +3895,7 @@ static void fts5FlushOneHash(Fts5Index *p){
/* The entire doclist will not fit on this leaf. The following
** loop iterates through the poslists that make up the current
** doclist. */
while( iOff<nDoclist ){
while( p->rc==SQLITE_OK && iOff<nDoclist ){
int nPos;
int nCopy;
int bDummy;
@ -3931,7 +3924,7 @@ static void fts5FlushOneHash(Fts5Index *p){
** that each varint must be stored contiguously. */
const u8 *pPoslist = &pDoclist[iOff];
int iPos = 0;
while( 1 ){
while( p->rc==SQLITE_OK ){
int nSpace = pgsz - pBuf->n;
int n = 0;
if( (nCopy - iPos)<=nSpace ){
@ -3990,7 +3983,7 @@ static void fts5FlushOneHash(Fts5Index *p){
*/
static void fts5IndexFlush(Fts5Index *p){
/* Unless it is empty, flush the hash table to disk */
if( p->rc==SQLITE_OK && p->nPendingData ){
if( p->nPendingData ){
assert( p->pHash );
p->nPendingData = 0;
fts5FlushOneHash(p);
@ -4337,6 +4330,7 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
}
/* Flush the hash table to disk if required */
if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
fts5IndexFlush(p);
}
@ -4932,7 +4926,6 @@ static void fts5DlidxIterTestReverse(
){
i64 iRowid = fts5DlidxIterRowid(pDlidx);
int pgno = fts5DlidxIterPgno(pDlidx);
assert( fts5DlidxIterPgno(pDlidx)>iLeaf );
cksum2 += iRowid + ((i64)pgno<<32);
}
@ -4971,23 +4964,31 @@ static void fts5IndexIntegrityCheckSegment(
if( pLeaf==0 ) break;
/* Check that the leaf contains at least one term, and that it is equal
** to or larger than the split-key in iter.term. */
** to or larger than the split-key in iter.term. Also check that if there
** is also a rowid pointer within the leaf page header, it points to a
** location before the term. */
iOff = fts5GetU16(&pLeaf->p[2]);
if( iOff==0 ){
p->rc = FTS5_CORRUPT;
}else{
int iRowidOff;
int nTerm; /* Size of term on leaf in bytes */
int res; /* Comparison of term and split-key */
iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
if( res==0 ) res = nTerm - iter.term.n;
if( res<0 ){
iRowidOff = fts5GetU16(&pLeaf->p[0]);
if( iRowidOff>=iOff ){
p->rc = FTS5_CORRUPT;
}else{
iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
if( res==0 ) res = nTerm - iter.term.n;
if( res<0 ) p->rc = FTS5_CORRUPT;
}
}
fts5DataRelease(pLeaf);
if( p->rc ) break;
/* Now check that the iter.nEmpty leaves following the current leaf
** (a) exist and (b) contain no terms. */
for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){

View File

@ -344,36 +344,36 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
pCsr->iCol = 0;
assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
while( 1 ){
while( rc==SQLITE_OK ){
const u8 *pPos; int nPos; /* Position list */
i64 iPos = 0; /* 64-bit position read from poslist */
int iOff = 0; /* Current offset within position list */
rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos);
if( rc!=SQLITE_OK ) break;
if( pTab->eType==FTS5_VOCAB_ROW ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aVal[1]++;
}
pCsr->aVal[0]++;
}else{
int iCol = -1;
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
int ii = FTS5_POS2COLUMN(iPos);
pCsr->aCnt[ii]++;
if( iCol!=ii ){
pCsr->aDoc[ii]++;
iCol = ii;
if( rc==SQLITE_OK ){
if( pTab->eType==FTS5_VOCAB_ROW ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aVal[1]++;
}
pCsr->aVal[0]++;
}else{
int iCol = -1;
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
int ii = FTS5_POS2COLUMN(iPos);
pCsr->aCnt[ii]++;
if( iCol!=ii ){
pCsr->aDoc[ii]++;
iCol = ii;
}
}
}
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
}
if( rc==SQLITE_OK ){
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
}
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
if( rc!=SQLITE_OK ) break;
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
}
}
}

View File

@ -31,6 +31,7 @@ do_execsql_test 1.0 {
}
set mask [expr 31 << 31]
if 1 {
# Test 1:
#
@ -114,6 +115,8 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
} {}
}
}
#-------------------------------------------------------------------------
# Test that corruption in leaf page headers is detected by queries that use
# doclist-indexes.
@ -130,17 +133,21 @@ do_execsql_test 3.0 {
foreach {tn hdr} {
1 "\x00\x00\x00\x00"
2 "\xFF\xFF\xFF\xFF"
3 "\x44\x45"
} {
set tn2 0
set nCorrupt 0
set nCorrupt2 0
foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
do_test 3.$tn.$tn2 {
do_test 3.$tn.$tn2.1 {
execsql BEGIN
set fd [db incrblob main x3_data block $rowid]
fconfigure $fd -encoding binary -translation binary
set existing [read $fd [string length $hdr]]
seek $fd 0
puts -nonewline $fd $hdr
close $fd
@ -149,6 +156,12 @@ foreach {tn hdr} {
set {} 1
} {1}
if {($tn2 % 10)==0 && $existing != $hdr} {
do_test 3.$tn.$tn2.2 {
catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
}
execsql ROLLBACK
}

View File

@ -26,11 +26,6 @@ proc do_fb_test {tn sql res} {
uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
# This test populates the FTS5 table containing $nEntry entries. Rows are
# numbered from 0 to ($nEntry-1). The rowid for row $i is:
#
@ -61,9 +56,9 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
}
execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
}
breakpoint
execsql COMMIT
breakpoint
do_test $tn.1 {
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
@ -81,12 +76,51 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
}
do_dlidx_test1 1.1 10 100 10000 0 1000
do_dlidx_test1 1.2 10 10 10000 0 128
do_dlidx_test1 1.3 10 10 66 0 36028797018963970
do_dlidx_test1 1.4 10 10 50 0 150000000000000000
foreach {tn pgsz} {
1 32
2 200
} {
do_execsql_test $tn.0 {
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
}
do_dlidx_test1 1.$tn.1 10 100 10000 0 1000
do_dlidx_test1 1.$tn.2 10 10 10000 0 128
do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970
do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000
do_dlidx_test1 1.$tn.5 10 10 200 0 [expr 1<<55]
do_dlidx_test1 1.$tn.6 10 10 30 0 [expr 1<<58]
}
proc do_dlidx_test2 {tn nEntry iFirst nStep} {
set str [string repeat "a " 500]
execsql {
BEGIN;
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
INSERT INTO t1 VALUES('b a');
WITH iii(ii, i) AS (
SELECT 1, $iFirst UNION ALL
SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry
)
INSERT INTO t1(rowid,x) SELECT i, $str FROM iii;
COMMIT;
}
do_execsql_test $tn.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
} {1}
breakpoint
do_execsql_test $tn.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
} {1}
}
do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
finish_test

View File

@ -317,14 +317,15 @@ do_execsql_test 10.0 {
COMMIT;
}
}
do_faultsim_test 10.1 -faults oom-t* -body {
db one { SELECT fts5_expr('a AND b NEAR(a b)') }
} -test {
faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}}
}
}
#do_faultsim_test 10.2 -faults oom-t* -body {
# db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') }
#} -test {
@ -333,6 +334,5 @@ do_faultsim_test 10.1 -faults oom-t* -body {
#}
finish_test

View File

@ -0,0 +1,96 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault5
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# OOM while creating an FTS5 table.
#
do_faultsim_test 1.1 -faults oom-t* -prep {
db eval { DROP TABLE IF EXISTS abc }
} -body {
db eval { CREATE VIRTUAL TABLE abc USING fts5(x,y) }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while writing a multi-tier doclist-index. And while running
# integrity-check on the same.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE tt USING fts5(x);
INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
}
faultsim_save_and_close
do_faultsim_test 2.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM tt }
} -body {
set str [string repeat "abc " 50]
db eval {
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
INSERT INTO tt(rowid, x) SELECT i, $str FROM ii;
}
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 2.2 -faults oom-t* -body {
db eval { INSERT INTO tt(tt) VALUES('integrity-check') }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while scanning an fts5vocab table.
#
reset_db
do_test 3.0 {
execsql {
CREATE VIRTUAL TABLE tt USING fts5(x);
CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
BEGIN;
}
for {set i 0} {$i < 20} {incr i} {
set str [string repeat "$i " 20]
execsql { INSERT INTO tt VALUES($str) }
}
execsql COMMIT
} {}
do_faultsim_test 2.1 -faults oom-t* -body {
db eval {
SELECT term FROM tv;
}
} -test {
faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}}
}
finish_test

View File

@ -1,5 +1,5 @@
C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
D 2015-05-13T18:12:58.028
C Add\stest\scases.\sAnd\ssome\sfixes.
D 2015-05-15T12:18:39.221
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in edfc69769e613a6359c42c06ea1d42c3bece1736
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -112,12 +112,12 @@ F ext/fts5/fts5_buffer.c 70b971e13503566f1e257941c60817ba0920a16b
F ext/fts5/fts5_config.c 05811f0bd80c396afcf3ceea68da16149a9a3258
F ext/fts5/fts5_expr.c 3fe1170453d6a322d2de8a3fd0aed3edff7b8b09
F ext/fts5/fts5_hash.c 54dd25348a46ea62ea96322c572e08cd1fb37304
F ext/fts5/fts5_index.c b9a3382af3027f5c9717d90613fda5f29f7d57fa
F ext/fts5/fts5_index.c 71d5ce47464f176e8708c7ec02d18613eb5eebda
F ext/fts5/fts5_storage.c cb8b585bfb7870a36101f1a8fa0b0777f4d1b68d
F ext/fts5/fts5_tcl.c f18eeb125d733f4e815a11679b971fa61cd7ec77
F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc
F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
F ext/fts5/fts5_vocab.c 2e37ea9b4d4d5460bc778f2adb872c6a869601e7
F ext/fts5/fts5_vocab.c e532f38a62d27fa662138a6cf33890b314225506
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
@ -138,15 +138,16 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
F ext/fts5/test/fts5content.test 532e15b541254410adc7bfb51f94631cfe82de8f
F ext/fts5/test/fts5corrupt.test 35bfdbbb3cdcea46ae7385f6432e9b5c574e70a1
F ext/fts5/test/fts5corrupt2.test bdad9241f73076917512f5298501d50d9e9d7dc7
F ext/fts5/test/fts5dlidx.test 74c3c8c33dfde594c0d8a22b9358d82fe56c8c7a
F ext/fts5/test/fts5corrupt2.test 88942d27ed581314f2867ef37352c72372c543df
F ext/fts5/test/fts5dlidx.test 070531bd45685e545e3e6021deb543f730a4011b
F ext/fts5/test/fts5doclist.test 635b80ac785627841a59c583bac702b55d49fdc5
F ext/fts5/test/fts5ea.test ed163ed820fd503354bd7dcf9d3b0e3801ade962
F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
F ext/fts5/test/fts5fault1.test b42d3296be8a75f557cf2cbce0d8b483fc9db45b
F ext/fts5/test/fts5fault2.test 26c3d70648f691e2cc9391e14bbc11a973656383
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
F ext/fts5/test/fts5fault4.test 420f2e23775b458eeb9a325bcdfe84650c2e9d39
F ext/fts5/test/fts5fault4.test a40e676e3e3b40901e2142f6fa5dff9e7313f5f7
F ext/fts5/test/fts5fault5.test 98e7e77bc1d8bb47c955e7d6dc870ab5736536e3
F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
F ext/fts5/test/fts5hash.test adb7b0442cc1c77c507f07e16d11490486e75dfa
F ext/fts5/test/fts5integrity.test 98801bd0fb7c53a40bc770280134865d61724f3a
@ -246,7 +247,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0
F src/complete.c a5cf5b4b56390cfb7b8636e8f7ddef90258dd575
F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
F src/date.c e4d50b3283696836ec1036b695ead9a19e37a5ac
F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a w src/test_stat.c
F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a
F src/delete.c 37964e6c1d73ff49cbea9ff690c9605fb15f600e
F src/expr.c 3fb2ab3ab69d15b4b75ae53fceb4e317f64cb306
F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
@ -1324,7 +1325,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P aa34bf666c384cf32a8d8166ab6d9afbca26a256 59e3e9e764440b7feaafadff74f422535d21bca2
R 7fb50d1391be18153fa78235920902bf
P b5f0e8c5b4bc018d672617ffd342d12b228548b9
R bdc9509f45810e2b38f3396d130ff7ee
U dan
Z 5ed330406879741f67800dc0f7082030
Z 2d21c26b247bac296f85906c81a77b9c

View File

@ -1 +1 @@
b5f0e8c5b4bc018d672617ffd342d12b228548b9
adee788586197445672013d434e7ba47ce510b59