mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add tests for fts5.

Browse Source 
FossilOrigin-Name: e748651c940eae2389fe826cf5c25f1166a5e611
This commit is contained in:
dan 2015-04-25 18:56:48 +00:00
parent ef8b74324d
commit b33e64c83e
8 changed files with 285 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

227
ext/fts5/fts5_index.c
View File

@ -256,6 +256,11 @@
*/
#define FTS5_SEGMENT_MAX_HEIGHT ((1 << FTS5_DATA_HEIGHT_B)-1)
/*
** Maximum segments permitted in a single index
*/
#define FTS5_MAX_SEGMENT 2000
/*
** The rowid for the doclist index associated with leaf page pgno of segment
** segid in index idx.
@ -365,6 +370,7 @@ struct Fts5StructureLevel {
};
struct Fts5Structure {
u64 nWriteCounter; /* Total leaves written to level 0 */
int nSegment; /* Total segments in this structure */
int nLevel; /* Number of levels in this index */
Fts5StructureLevel aLevel[0]; /* Array of nLevel level objects */
};
@ -725,6 +731,7 @@ static int fts5BufferCompareBlob(
return (res==0 ? (pLeft->n - nRight) : res);
}
/*
** Compare the contents of the two buffers using memcmp(). If one buffer
** is a prefix of the other, it is considered the lesser.
@ -740,6 +747,17 @@ static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
return (res==0 ? (pLeft->n - pRight->n) : res);
}
#ifdef SQLITE_DEBUG
static int fts5BlobCompare(
const u8 *pLeft, int nLeft,
const u8 *pRight, int nRight
){
int nCmp = MIN(nLeft, nRight);
int res = memcmp(pLeft, pRight, nCmp);
return (res==0 ? (nLeft - nRight) : res);
}
#endif
/*
** Close the read-only blob handle, if it is open.
@ -1042,6 +1060,7 @@ static int fts5StructureDecode(
if( pRet ){
pRet->nLevel = nLevel;
pRet->nSegment = nSegment;
i += sqlite3GetVarint(&pData[i], &pRet->nWriteCounter);
for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){
@ -1166,8 +1185,10 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){
}
/*
** Return the total number of segments in index structure pStruct.
** Return the total number of segments in index structure pStruct. This
** function is only ever used as part of assert() conditions.
*/
#ifdef SQLITE_DEBUG
static int fts5StructureCountSegments(Fts5Structure *pStruct){
int nSegment = 0; /* Total number of segments */
if( pStruct ){
@ -1179,6 +1200,7 @@ static int fts5StructureCountSegments(Fts5Structure *pStruct){
return nSegment;
}
#endif
/*
** Serialize and store the "structure" record for index iIdx.
@ -1188,12 +1210,11 @@ static int fts5StructureCountSegments(Fts5Structure *pStruct){
*/
static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
if( p->rc==SQLITE_OK ){
int nSegment; /* Total number of segments */
Fts5Buffer buf; /* Buffer to serialize record into */
int iLvl; /* Used to iterate through levels */
int iCookie; /* Cookie value to store */
nSegment = fts5StructureCountSegments(pStruct);
assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
memset(&buf, 0, sizeof(Fts5Buffer));
/* Append the current configuration cookie */
@ -1202,7 +1223,7 @@ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
fts5BufferAppend32(&p->rc, &buf, iCookie);
fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel);
fts5BufferAppendVarint(&p->rc, &buf, nSegment);
fts5BufferAppendVarint(&p->rc, &buf, pStruct->nSegment);
fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter);
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
@ -1583,6 +1604,7 @@ static void fts5SegIterNextPage(
static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
int nSz;
int n = fts5GetVarint32(p, nSz);
assert_nc( nSz>=0 );
*pnSz = nSz/2;
*pbDel = nSz & 0x0001;
return n;
@ -1601,9 +1623,13 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
*/
static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
if( p->rc==SQLITE_OK ){
const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
int iOff = pIter->iLeafOffset; /* Offset to read at */
pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel);
if( iOff>=pIter->pLeaf->n ){
p->rc = FTS5_CORRUPT;
}else{
const u8 *a = &pIter->pLeaf->p[iOff];
pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel);
}
}
}
@ -2028,18 +2054,16 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
int iOff = pIter->iLeafOffset + pIter->nPos;
while( iOff<pLeaf->n ){
int bDummy;
int nPos;
i64 iDelta;
/* iOff is currently the offset of the start of position list data */
iOff += getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
if( iDelta==0 ) return;
if( iOff<pLeaf->n ){
int bDummy;
int nPos;
iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy);
iOff += nPos;
}
assert_nc( iOff<pLeaf->n );
iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy);
iOff += nPos;
}
}
@ -2346,11 +2370,9 @@ static void fts5SegIterGotoPage(
int iLeafPgno
){
assert( iLeafPgno>pIter->iLeafPgno );
if( p->rc==SQLITE_OK ){
pIter->iLeafPgno = iLeafPgno-1;
fts5SegIterNextPage(p, pIter);
assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
}
pIter->iLeafPgno = iLeafPgno-1;
fts5SegIterNextPage(p, pIter);
assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
if( p->rc==SQLITE_OK ){
int iOff;
@ -2413,7 +2435,7 @@ static void fts5SegIterNextFrom(
}
}
while( 1 ){
while( p->rc==SQLITE_OK ){
if( bMove ) fts5SegIterNext(p, pIter, 0);
if( pIter->pLeaf==0 ) break;
if( bRev==0 && pIter->iRowid>=iMatch ) break;
@ -2502,6 +2524,7 @@ static void fts5MultiIterNext(
int iFirst = pIter->aFirst[1].iFirst;
int bNewTerm = 0;
Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
assert( p->rc==SQLITE_OK );
if( bUseFrom && pSeg->pDlidx ){
fts5SegIterNextFrom(p, pSeg, iFrom);
}else{
@ -2542,8 +2565,8 @@ static void fts5MultiIterNew(
int nSegment, /* Number of segments to merge (iLevel>=0) */
Fts5MultiSegIter **ppOut /* New object */
){
int nSeg; /* Number of segments merged */
int nSlot; /* Power of two >= nSeg */
int nSeg; /* Number of segment-iters in use */
int nSlot = 0; /* Power of two >= nSeg */
int iIter = 0; /* */
int iSeg; /* Used to iterate through segments */
Fts5StructureLevel *pLvl;
@ -2552,13 +2575,17 @@ static void fts5MultiIterNew(
assert( (pTerm==0 && nTerm==0) || iLevel<0 );
/* Allocate space for the new multi-seg-iterator. */
if( iLevel<0 ){
nSeg = fts5StructureCountSegments(pStruct);
nSeg += (p->apHash ? 1 : 0);
}else{
nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
if( p->rc==SQLITE_OK ){
if( iLevel<0 ){
assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
nSeg = pStruct->nSegment;
nSeg += (p->apHash ? 1 : 0);
}else{
nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
}
for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
}
for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
*ppOut = pNew = fts5IdxMalloc(p,
sizeof(Fts5MultiSegIter) + /* pNew */
sizeof(Fts5SegIter) * nSlot + /* pNew->aSeg[] */
@ -2732,101 +2759,41 @@ static void fts5ChunkIterRelease(Fts5ChunkIter *pIter){
pIter->pLeaf = 0;
}
/*
** Read and return the next 32-bit varint from the position-list iterator
** passed as the second argument.
**
** If an error occurs, zero is returned an an error code left in
** Fts5Index.rc. If an error has already occurred when this function is
** called, it is a no-op.
*/
static int fts5PosIterReadVarint(Fts5Index *p, Fts5PosIter *pIter){
int iVal = 0;
if( p->rc==SQLITE_OK ){
if( pIter->iOff>=pIter->chunk.n ){
fts5ChunkIterNext(p, &pIter->chunk);
if( fts5ChunkIterEof(p, &pIter->chunk) ) return 0;
pIter->iOff = 0;
}
pIter->iOff += fts5GetVarint32(&pIter->chunk.p[pIter->iOff], iVal);
}
return iVal;
}
/*
** Advance the position list iterator to the next entry.
*/
static void fts5PosIterNext(Fts5Index *p, Fts5PosIter *pIter){
int iVal;
assert( fts5ChunkIterEof(p, &pIter->chunk)==0 );
iVal = fts5PosIterReadVarint(p, pIter);
if( fts5ChunkIterEof(p, &pIter->chunk)==0 ){
if( iVal==1 ){
pIter->iCol = fts5PosIterReadVarint(p, pIter);
pIter->iPos = fts5PosIterReadVarint(p, pIter) - 2;
}else{
pIter->iPos += (iVal - 2);
}
}
}
/*
** Initialize the Fts5PosIter object passed as the final argument to iterate
** through the position-list associated with the index entry that iterator
** pMulti currently points to.
*/
static void fts5PosIterInit(
Fts5Index *p, /* FTS5 backend object */
Fts5MultiSegIter *pMulti, /* Multi-seg iterator to read pos-list from */
Fts5PosIter *pIter /* Initialize this object */
){
if( p->rc==SQLITE_OK ){
Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
memset(pIter, 0, sizeof(*pIter));
fts5ChunkIterInit(p, pSeg, &pIter->chunk);
if( fts5ChunkIterEof(p, &pIter->chunk)==0 ){
fts5PosIterNext(p, pIter);
}
}
}
/*
** Return true if the position iterator passed as the second argument is
** at EOF. Or if an error has already occurred. Otherwise, return false.
*/
static int fts5PosIterEof(Fts5Index *p, Fts5PosIter *pIter){
return (p->rc || pIter->chunk.pLeaf==0);
}
/*
** Allocate a new segment-id for the structure pStruct.
** Allocate a new segment-id for the structure pStruct. The new segment
** id must be between 1 and 65335 inclusive, and must not be used by
** any currently existing segment. If a free segment id cannot be found,
** SQLITE_FULL is returned.
**
** If an error has already occurred, this function is a no-op. 0 is
** returned in this case.
*/
static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
int i;
if( p->rc!=SQLITE_OK ) return 0;
u32 iSegid = 0;
for(i=0; i<100; i++){
int iSegid;
sqlite3_randomness(sizeof(int), (void*)&iSegid);
iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
if( iSegid ){
int iLvl, iSeg;
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
iSegid = 0;
if( p->rc==SQLITE_OK ){
if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
p->rc = SQLITE_FULL;
}else{
while( iSegid==0 ){
int iLvl, iSeg;
sqlite3_randomness(sizeof(u32), (void*)&iSegid);
iSegid = (iSegid % ((1 << FTS5_DATA_ID_B) - 2)) + 1;
assert( iSegid>0 && iSegid<=65535 );
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
iSegid = 0;
}
}
}
}
}
if( iSegid ) return iSegid;
}
p->rc = SQLITE_ERROR;
return 0;
return (int)iSegid;
}
/*
@ -2838,7 +2805,7 @@ static void fts5IndexDiscardData(Fts5Index *p){
Fts5Config *pConfig = p->pConfig;
int i;
for(i=0; i<=pConfig->nPrefix; i++){
if( p->apHash[i] ) sqlite3Fts5HashClear(p->apHash[i]);
sqlite3Fts5HashClear(p->apHash[i]);
}
p->nPendingData = 0;
}
@ -2853,7 +2820,8 @@ static int fts5PrefixCompress(
int nNew, const u8 *pNew
){
int i;
for(i=0; i<nNew && i<nOld; i++){
assert( fts5BlobCompare(pOld, nOld, pNew, nNew)<0 );
for(i=0; i<nOld; i++){
if( pOld[i]!=pNew[i] ) break;
}
return i;
@ -2861,14 +2829,14 @@ static int fts5PrefixCompress(
/*
** If an "nEmpty" record must be written to the b-tree before the next
** term, write it now.
** term, write it now.
*/
static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
if( pWriter->nEmpty ){
int bFlag = 0;
Fts5PageWriter *pPg;
pPg = &pWriter->aWriter[1];
if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE && pWriter->cdlidx.n ){
if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
i64 iKey = FTS5_DOCLIST_IDX_ROWID(
pWriter->iIdx, pWriter->iSegid,
pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty
@ -3040,8 +3008,8 @@ static void fts5WriteAppendTerm(
int nPrefix; /* Bytes of prefix compression for term */
Fts5PageWriter *pPage = &pWriter->aWriter[0];
assert( pPage==0 || pPage->buf.n==0 || pPage->buf.n>4 );
if( pPage && pPage->buf.n==0 ){
assert( pPage->buf.n==0 || pPage->buf.n>4 );
if( pPage->buf.n==0 ){
/* Zero the first term and first docid fields */
static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
@ -3137,6 +3105,7 @@ static void fts5WriteAppendRowid(
}
}
#if 0
static void fts5WriteAppendPoslistInt(
Fts5Index *p,
Fts5SegWriter *pWriter,
@ -3150,6 +3119,7 @@ static void fts5WriteAppendPoslistInt(
}
}
}
#endif
static void fts5WriteAppendPoslistData(
Fts5Index *p,
@ -3389,6 +3359,7 @@ static void fts5IndexMergeLevel(
pLvlOut->nSeg++;
pSeg->pgnoFirst = 1;
pSeg->iSegid = iSegid;
pStruct->nSegment++;
/* Read input from all segments in the input level */
nInput = pLvl->nSeg;
@ -3461,10 +3432,12 @@ fflush(stdout);
int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment);
memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove);
}
pStruct->nSegment -= nInput;
pLvl->nSeg -= nInput;
pLvl->nMerge = 0;
if( pSeg->pgnoLast==0 ){
pLvlOut->nSeg--;
pStruct->nSegment--;
}
}else{
assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 );
@ -3768,6 +3741,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
pSeg->nHeight = nHeight;
pSeg->pgnoFirst = 1;
pSeg->pgnoLast = pgnoLast;
pStruct->nSegment++;
}
fts5StructurePromote(p, 0, pStruct);
}
@ -3809,7 +3783,8 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
Fts5Structure *pNew = 0;
int nSeg = 0;
if( pStruct ){
nSeg = fts5StructureCountSegments(pStruct);
assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
nSeg = pStruct->nSegment;
if( nSeg>1 ){
int nByte = sizeof(Fts5Structure);
nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
@ -3832,7 +3807,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
iSegOut++;
}
}
pLvl->nSeg = nSeg;
pNew->nSegment = pLvl->nSeg = nSeg;
}else{
sqlite3_free(pNew);
pNew = 0;
@ -4146,7 +4121,11 @@ static void fts5MultiIterPoslist(
Fts5ChunkIter iter;
Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
assert( fts5MultiIterEof(p, pMulti)==0 );
static int nCall = 0;
nCall++;
fts5ChunkIterInit(p, pSeg, &iter);
if( fts5ChunkIterEof(p, &iter)==0 ){
if( bSz ){
/* WRITEPOSLISTSIZE */
@ -4434,6 +4413,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
u64 cksum2 = 0; /* Checksum based on contents of indexes */
u64 cksum3 = 0; /* Checksum based on contents of indexes */
Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */
Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
/* Check that the internal nodes of each segment match the leaves */
for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
@ -4470,18 +4450,18 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
fts5MultiIterEof(p, pIter)==0;
fts5MultiIterNext(p, pIter, 0, 0)
){
Fts5PosIter sPos; /* Used to iterate through position list */
int n; /* Size of term in bytes */
i64 iPos = 0; /* Position read from poslist */
int iOff = 0; /* Offset within poslist */
i64 iRowid = fts5MultiIterRowid(pIter);
char *z = (char*)fts5MultiIterTerm(pIter, &n);
/* Update cksum2 with the entries associated with the current term
** and rowid. */
for(fts5PosIterInit(p, pIter, &sPos);
fts5PosIterEof(p, &sPos)==0;
fts5PosIterNext(p, &sPos)
){
cksum2 ^= fts5IndexEntryCksum(iRowid, sPos.iCol, sPos.iPos, z, n);
poslist.n = 0;
fts5MultiIterPoslist(p, pIter, 0, &poslist);
while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
int iCol = FTS5_POS2COLUMN(iPos);
int iTokOff = FTS5_POS2OFFSET(iPos);
cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n);
}
/* If this is a new term, query for it. Update cksum3 with the results. */
@ -4526,6 +4506,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT;
fts5BufferFree(&term);
fts5BufferFree(&poslist);
return fts5IndexReturn(p);
}

8
ext/fts5/test/fts5aa.test
View File

@ -47,11 +47,9 @@ do_execsql_test 2.0 {
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
}
do_execsql_test 2.2 {
SELECT fts5_decode(id, block) FROM t1_data WHERE id==10
} {
{{structure idx=0} {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}}
}
do_test 2.2 {
execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
} {/{{structure idx=0} {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/}
do_execsql_test 2.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
}

22
ext/fts5/test/fts5ab.test
View File

@ -263,5 +263,27 @@ do_execsql_test 6.3 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a'
} {1 2}
do_test 6.4 {
db close
sqlite3 db test.db
execsql {
BEGIN;
INSERT INTO s3(s3) VALUES('optimize');
ROLLBACK;
}
} {}
#-------------------------------------------------------------------------
#
set doc [string repeat "a b c " 500]
breakpoint
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
INSERT INTO x1 VALUES($doc);
}
finish_test

86
ext/fts5/test/fts5corrupt2.test
View File

@ -28,7 +28,6 @@ do_execsql_test 1.0 {
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
set mask [expr 31 << 31]
# Test 1:
@ -51,7 +50,6 @@ foreach {tno stmt} {
1 { DELETE FROM t1_data WHERE rowid=$rowid }
2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid }
} {
break
set tn 0
foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] {
incr tn
@ -77,6 +75,8 @@ foreach {tno stmt} {
}
}
# Using the same database as the 1.* tests.
#
# Run N-1 tests, where N is the number of bytes in the rightmost leaf page
# of the fts index. For test $i, truncate the rightmost leafpage to $i
# bytes. Then test both the integrity-check detects the corruption.
@ -112,5 +112,87 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
} {}
}
#-------------------------------------------------------------------------
# Test that corruption in leaf page headers is detected by queries that use
# doclist-indexes.
#
set doc "A B C D E F G H I J "
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE x3 USING fts5(tt);
INSERT INTO x3(x3, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000)
INSERT INTO x3
SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
}
foreach {tn hdr} {
1 "\00\00\00\00"
2 "\FF\FF\FF\FF"
} {
set tn2 0
set nCorrupt 0
foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
do_test 3.$tn.$tn2 {
execsql BEGIN
set fd [db incrblob main x3_data block $rowid]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd $hdr
close $fd
set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}]
if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
set {} 1
} {1}
execsql ROLLBACK
}
do_test 3.$tn.x { expr $nCorrupt>0 } 1
}
#--------------------------------------------------------------------
#
set doc "A B C D E F G H I J "
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x4 USING fts5(tt);
INSERT INTO x4(x4, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)
INSERT INTO x4
SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
}
foreach {tn nCut} {
1 1
2 10
} {
set tn2 0
set nCorrupt 0
foreach rowid [db eval {SELECT rowid FROM x4_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
do_test 4.$tn.$tn2 {
execsql {
BEGIN;
UPDATE x4_data SET block = substr(block, 1, length(block)-$nCut)
WHERE id = $rowid;
}
set res [catchsql {
SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC
}]
if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
set {} 1
} {1}
execsql ROLLBACK
}
do_test 4.$tn.x { expr $nCorrupt>0 } 1
}
finish_test

23
ext/fts5/test/fts5fault2.test
View File

@ -51,5 +51,28 @@ do_faultsim_test 1.2 -faults oom-* -prep {
faultsim_test_result {0 {1000 900 800 700 600 500 400 300 200 100}}
}
#-------------------------------------------------------------------------
# OOM within a query that accesses the in-memory hash table.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE "a b c" USING fts5(a, b, c);
INSERT INTO "a b c" VALUES('one two', 'x x x', 'three four');
INSERT INTO "a b c" VALUES('nine ten', 'y y y', 'two two');
}
do_faultsim_test 2.1 -faults oom-trans* -prep {
execsql {
BEGIN;
INSERT INTO "a b c" VALUES('one one', 'z z z', 'nine ten');
}
} -body {
execsql { SELECT rowid FROM "a b c" WHERE "a b c" MATCH 'one' }
} -test {
faultsim_test_result {0 {1 3}}
catchsql { ROLLBACK }
}
finish_test

37
ext/fts5/test/fts5full.test Normal file
View File

@ -0,0 +1,37 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Test that SQLITE_FULL is returned if the FTS5 table cannot find a free
# segid to use. In practice this can only really happen when automerge and
# crisismerge are both disabled.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5full
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE x8 USING fts5(i);
INSERT INTO x8(x8, rank) VALUES('automerge', 0);
INSERT INTO x8(x8, rank) VALUES('crisismerge', 100000);
}
db func rnddoc fts5_rnddoc
do_test 1.1 {
list [catch {
for {set i 0} {$i < 2500} {incr i} {
execsql { INSERT INTO x8 VALUES( rnddoc(5) ); }
}
} msg] $msg
} {1 {database or disk is full}}
finish_test

21
manifest
View File

@ -1,5 +1,5 @@
C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
D 2015-04-24T20:18:21.844
C Add\stests\sfor\sfts5.
D 2015-04-25T18:56:48.351
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23
F ext/fts5/fts5_config.c 43fcf838d3a3390d1245e3d5e651fa5cc1df575b
F ext/fts5/fts5_expr.c 05da381ab26031243266069302c6eb4094b2c5dd
F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529
F ext/fts5/fts5_index.c 39810b25a017f2626ac72b3e44afe9b534e5d5db
F ext/fts5/fts5_index.c c87369d11271847df9f033f0df148e7f004a88a2
F ext/fts5/fts5_storage.c b3a4cbbcd197fe587789398e51a631f92fc9196c
F ext/fts5/fts5_tcl.c 10bf0eb678d34c1bfdcfaf653d2e6dd92afa8b38
F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
@ -120,8 +120,8 @@ F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
F ext/fts5/test/fts5aa.test 91f22b3cc7b372a2903c828e907a1e52f1177b8a
F ext/fts5/test/fts5ab.test 5da2e92a8047860b9e22b6fd3990549639d631b1
F ext/fts5/test/fts5aa.test 87b2e01084c3e2a960ae1500dd5f0729dac2166c
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e
F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d
F ext/fts5/test/fts5ae.test 9175201baf8c885fc1cbb2da11a0c61fd11224db
@ -136,12 +136,13 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517
F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727
F ext/fts5/test/fts5corrupt2.test 3be48d8a30d30e3ae819f04e957c45d091bfbb85
F ext/fts5/test/fts5corrupt2.test 494111fd4f2dab36499cf97718eaba1f7c11e9d0
F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f
F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
F ext/fts5/test/fts5fault1.test ed71717a479bef32d05f02d9c48691011d160d4d
F ext/fts5/test/fts5fault2.test 000ecebf28b8f2cd520f44c70962625ee11d65ac
F ext/fts5/test/fts5fault2.test f478fa94e39a6911189f9e052a3b93ab4cd275fa
F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
@ -1304,7 +1305,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 86309961344f4076ddcf55d730d3600ec3b6e45c dc88fe7e641c52d27fba8c753cee590db87388c5
R e0fc7874fca6c46f36b4c18c43a12035
P 1c78d8920fb59da3cb97dd2eb09b3e08dfd14259
R 6572bfe0dee7d9becde3a8bb0a8d33a2
U dan
Z 339a1949f9a5bc9b03407831c5b374c4
Z e7bb2196ce75c57e787365bfa027743a

2
manifest.uuid
View File

@ -1 +1 @@
1c78d8920fb59da3cb97dd2eb09b3e08dfd14259
e748651c940eae2389fe826cf5c25f1166a5e611