From b33e64c83e8788f958f5786dfcd83179bfbe4274 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Apr 2015 18:56:48 +0000 Subject: [PATCH] Add tests for fts5. FossilOrigin-Name: e748651c940eae2389fe826cf5c25f1166a5e611 --- ext/fts5/fts5_index.c | 227 +++++++++++++++----------------- ext/fts5/test/fts5aa.test | 8 +- ext/fts5/test/fts5ab.test | 22 ++++ ext/fts5/test/fts5corrupt2.test | 86 +++++++++++- ext/fts5/test/fts5fault2.test | 23 ++++ ext/fts5/test/fts5full.test | 37 ++++++ manifest | 21 +-- manifest.uuid | 2 +- 8 files changed, 285 insertions(+), 141 deletions(-) create mode 100644 ext/fts5/test/fts5full.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index c109cff57d..746e44bdbb 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -256,6 +256,11 @@ */ #define FTS5_SEGMENT_MAX_HEIGHT ((1 << FTS5_DATA_HEIGHT_B)-1) +/* +** Maximum segments permitted in a single index +*/ +#define FTS5_MAX_SEGMENT 2000 + /* ** The rowid for the doclist index associated with leaf page pgno of segment ** segid in index idx. @@ -365,6 +370,7 @@ struct Fts5StructureLevel { }; struct Fts5Structure { u64 nWriteCounter; /* Total leaves written to level 0 */ + int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[0]; /* Array of nLevel level objects */ }; @@ -725,6 +731,7 @@ static int fts5BufferCompareBlob( return (res==0 ? (pLeft->n - nRight) : res); } + /* ** Compare the contents of the two buffers using memcmp(). If one buffer ** is a prefix of the other, it is considered the lesser. @@ -740,6 +747,17 @@ static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ return (res==0 ? (pLeft->n - pRight->n) : res); } +#ifdef SQLITE_DEBUG +static int fts5BlobCompare( + const u8 *pLeft, int nLeft, + const u8 *pRight, int nRight +){ + int nCmp = MIN(nLeft, nRight); + int res = memcmp(pLeft, pRight, nCmp); + return (res==0 ? (nLeft - nRight) : res); +} +#endif + /* ** Close the read-only blob handle, if it is open. @@ -1042,6 +1060,7 @@ static int fts5StructureDecode( if( pRet ){ pRet->nLevel = nLevel; + pRet->nSegment = nSegment; i += sqlite3GetVarint(&pData[i], &pRet->nWriteCounter); for(iLvl=0; rc==SQLITE_OK && iLvlrc==SQLITE_OK ){ - int nSegment; /* Total number of segments */ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ - nSegment = fts5StructureCountSegments(pStruct); + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); memset(&buf, 0, sizeof(Fts5Buffer)); /* Append the current configuration cookie */ @@ -1202,7 +1223,7 @@ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){ fts5BufferAppend32(&p->rc, &buf, iCookie); fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel); - fts5BufferAppendVarint(&p->rc, &buf, nSegment); + fts5BufferAppendVarint(&p->rc, &buf, pStruct->nSegment); fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter); for(iLvl=0; iLvlnLevel; iLvl++){ @@ -1583,6 +1604,7 @@ static void fts5SegIterNextPage( static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ int nSz; int n = fts5GetVarint32(p, nSz); + assert_nc( nSz>=0 ); *pnSz = nSz/2; *pbDel = nSz & 0x0001; return n; @@ -1601,9 +1623,13 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ - const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; int iOff = pIter->iLeafOffset; /* Offset to read at */ - pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel); + if( iOff>=pIter->pLeaf->n ){ + p->rc = FTS5_CORRUPT; + }else{ + const u8 *a = &pIter->pLeaf->p[iOff]; + pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel); + } } } @@ -2028,18 +2054,16 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){ if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ int iOff = pIter->iLeafOffset + pIter->nPos; while( iOffn ){ + int bDummy; + int nPos; i64 iDelta; /* iOff is currently the offset of the start of position list data */ iOff += getVarint(&pLeaf->p[iOff], (u64*)&iDelta); if( iDelta==0 ) return; - - if( iOffn ){ - int bDummy; - int nPos; - iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); - iOff += nPos; - } + assert_nc( iOffn ); + iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); + iOff += nPos; } } @@ -2346,11 +2370,9 @@ static void fts5SegIterGotoPage( int iLeafPgno ){ assert( iLeafPgno>pIter->iLeafPgno ); - if( p->rc==SQLITE_OK ){ - pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - } + pIter->iLeafPgno = iLeafPgno-1; + fts5SegIterNextPage(p, pIter); + assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); if( p->rc==SQLITE_OK ){ int iOff; @@ -2413,7 +2435,7 @@ static void fts5SegIterNextFrom( } } - while( 1 ){ + while( p->rc==SQLITE_OK ){ if( bMove ) fts5SegIterNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; @@ -2502,6 +2524,7 @@ static void fts5MultiIterNext( int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + assert( p->rc==SQLITE_OK ); if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ @@ -2542,8 +2565,8 @@ static void fts5MultiIterNew( int nSegment, /* Number of segments to merge (iLevel>=0) */ Fts5MultiSegIter **ppOut /* New object */ ){ - int nSeg; /* Number of segments merged */ - int nSlot; /* Power of two >= nSeg */ + int nSeg; /* Number of segment-iters in use */ + int nSlot = 0; /* Power of two >= nSeg */ int iIter = 0; /* */ int iSeg; /* Used to iterate through segments */ Fts5StructureLevel *pLvl; @@ -2552,13 +2575,17 @@ static void fts5MultiIterNew( assert( (pTerm==0 && nTerm==0) || iLevel<0 ); /* Allocate space for the new multi-seg-iterator. */ - if( iLevel<0 ){ - nSeg = fts5StructureCountSegments(pStruct); - nSeg += (p->apHash ? 1 : 0); - }else{ - nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + nSeg += (p->apHash ? 1 : 0); + }else{ + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + } + for(nSlot=2; nSlotaSeg[] */ @@ -2732,101 +2759,41 @@ static void fts5ChunkIterRelease(Fts5ChunkIter *pIter){ pIter->pLeaf = 0; } -/* -** Read and return the next 32-bit varint from the position-list iterator -** passed as the second argument. -** -** If an error occurs, zero is returned an an error code left in -** Fts5Index.rc. If an error has already occurred when this function is -** called, it is a no-op. -*/ -static int fts5PosIterReadVarint(Fts5Index *p, Fts5PosIter *pIter){ - int iVal = 0; - if( p->rc==SQLITE_OK ){ - if( pIter->iOff>=pIter->chunk.n ){ - fts5ChunkIterNext(p, &pIter->chunk); - if( fts5ChunkIterEof(p, &pIter->chunk) ) return 0; - pIter->iOff = 0; - } - pIter->iOff += fts5GetVarint32(&pIter->chunk.p[pIter->iOff], iVal); - } - return iVal; -} /* -** Advance the position list iterator to the next entry. -*/ -static void fts5PosIterNext(Fts5Index *p, Fts5PosIter *pIter){ - int iVal; - assert( fts5ChunkIterEof(p, &pIter->chunk)==0 ); - iVal = fts5PosIterReadVarint(p, pIter); - if( fts5ChunkIterEof(p, &pIter->chunk)==0 ){ - if( iVal==1 ){ - pIter->iCol = fts5PosIterReadVarint(p, pIter); - pIter->iPos = fts5PosIterReadVarint(p, pIter) - 2; - }else{ - pIter->iPos += (iVal - 2); - } - } -} - -/* -** Initialize the Fts5PosIter object passed as the final argument to iterate -** through the position-list associated with the index entry that iterator -** pMulti currently points to. -*/ -static void fts5PosIterInit( - Fts5Index *p, /* FTS5 backend object */ - Fts5MultiSegIter *pMulti, /* Multi-seg iterator to read pos-list from */ - Fts5PosIter *pIter /* Initialize this object */ -){ - if( p->rc==SQLITE_OK ){ - Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; - memset(pIter, 0, sizeof(*pIter)); - fts5ChunkIterInit(p, pSeg, &pIter->chunk); - if( fts5ChunkIterEof(p, &pIter->chunk)==0 ){ - fts5PosIterNext(p, pIter); - } - } -} - -/* -** Return true if the position iterator passed as the second argument is -** at EOF. Or if an error has already occurred. Otherwise, return false. -*/ -static int fts5PosIterEof(Fts5Index *p, Fts5PosIter *pIter){ - return (p->rc || pIter->chunk.pLeaf==0); -} - -/* -** Allocate a new segment-id for the structure pStruct. +** Allocate a new segment-id for the structure pStruct. The new segment +** id must be between 1 and 65335 inclusive, and must not be used by +** any currently existing segment. If a free segment id cannot be found, +** SQLITE_FULL is returned. ** ** If an error has already occurred, this function is a no-op. 0 is ** returned in this case. */ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ int i; - if( p->rc!=SQLITE_OK ) return 0; + u32 iSegid = 0; - for(i=0; i<100; i++){ - int iSegid; - sqlite3_randomness(sizeof(int), (void*)&iSegid); - iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); - if( iSegid ){ - int iLvl, iSeg; - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ - iSegid = 0; + if( p->rc==SQLITE_OK ){ + if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ + p->rc = SQLITE_FULL; + }else{ + while( iSegid==0 ){ + int iLvl, iSeg; + sqlite3_randomness(sizeof(u32), (void*)&iSegid); + iSegid = (iSegid % ((1 << FTS5_DATA_ID_B) - 2)) + 1; + assert( iSegid>0 && iSegid<=65535 ); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ + iSegid = 0; + } } } } } - if( iSegid ) return iSegid; } - p->rc = SQLITE_ERROR; - return 0; + return (int)iSegid; } /* @@ -2838,7 +2805,7 @@ static void fts5IndexDiscardData(Fts5Index *p){ Fts5Config *pConfig = p->pConfig; int i; for(i=0; i<=pConfig->nPrefix; i++){ - if( p->apHash[i] ) sqlite3Fts5HashClear(p->apHash[i]); + sqlite3Fts5HashClear(p->apHash[i]); } p->nPendingData = 0; } @@ -2853,7 +2820,8 @@ static int fts5PrefixCompress( int nNew, const u8 *pNew ){ int i; - for(i=0; inEmpty ){ int bFlag = 0; Fts5PageWriter *pPg; pPg = &pWriter->aWriter[1]; - if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE && pWriter->cdlidx.n ){ + if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ i64 iKey = FTS5_DOCLIST_IDX_ROWID( pWriter->iIdx, pWriter->iSegid, pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty @@ -3040,8 +3008,8 @@ static void fts5WriteAppendTerm( int nPrefix; /* Bytes of prefix compression for term */ Fts5PageWriter *pPage = &pWriter->aWriter[0]; - assert( pPage==0 || pPage->buf.n==0 || pPage->buf.n>4 ); - if( pPage && pPage->buf.n==0 ){ + assert( pPage->buf.n==0 || pPage->buf.n>4 ); + if( pPage->buf.n==0 ){ /* Zero the first term and first docid fields */ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); @@ -3137,6 +3105,7 @@ static void fts5WriteAppendRowid( } } +#if 0 static void fts5WriteAppendPoslistInt( Fts5Index *p, Fts5SegWriter *pWriter, @@ -3150,6 +3119,7 @@ static void fts5WriteAppendPoslistInt( } } } +#endif static void fts5WriteAppendPoslistData( Fts5Index *p, @@ -3389,6 +3359,7 @@ static void fts5IndexMergeLevel( pLvlOut->nSeg++; pSeg->pgnoFirst = 1; pSeg->iSegid = iSegid; + pStruct->nSegment++; /* Read input from all segments in the input level */ nInput = pLvl->nSeg; @@ -3461,10 +3432,12 @@ fflush(stdout); int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove); } + pStruct->nSegment -= nInput; pLvl->nSeg -= nInput; pLvl->nMerge = 0; if( pSeg->pgnoLast==0 ){ pLvlOut->nSeg--; + pStruct->nSegment--; } }else{ assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 ); @@ -3768,6 +3741,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){ pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; } fts5StructurePromote(p, 0, pStruct); } @@ -3809,7 +3783,8 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ Fts5Structure *pNew = 0; int nSeg = 0; if( pStruct ){ - nSeg = fts5StructureCountSegments(pStruct); + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; if( nSeg>1 ){ int nByte = sizeof(Fts5Structure); nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); @@ -3832,7 +3807,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ iSegOut++; } } - pLvl->nSeg = nSeg; + pNew->nSegment = pLvl->nSeg = nSeg; }else{ sqlite3_free(pNew); pNew = 0; @@ -4146,7 +4121,11 @@ static void fts5MultiIterPoslist( Fts5ChunkIter iter; Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); + static int nCall = 0; + nCall++; + fts5ChunkIterInit(p, pSeg, &iter); + if( fts5ChunkIterEof(p, &iter)==0 ){ if( bSz ){ /* WRITEPOSLISTSIZE */ @@ -4434,6 +4413,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum2 = 0; /* Checksum based on contents of indexes */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ + Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ /* Check that the internal nodes of each segment match the leaves */ for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){ @@ -4470,18 +4450,18 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ - Fts5PosIter sPos; /* Used to iterate through position list */ int n; /* Size of term in bytes */ + i64 iPos = 0; /* Position read from poslist */ + int iOff = 0; /* Offset within poslist */ i64 iRowid = fts5MultiIterRowid(pIter); char *z = (char*)fts5MultiIterTerm(pIter, &n); - /* Update cksum2 with the entries associated with the current term - ** and rowid. */ - for(fts5PosIterInit(p, pIter, &sPos); - fts5PosIterEof(p, &sPos)==0; - fts5PosIterNext(p, &sPos) - ){ - cksum2 ^= fts5IndexEntryCksum(iRowid, sPos.iCol, sPos.iPos, z, n); + poslist.n = 0; + fts5MultiIterPoslist(p, pIter, 0, &poslist); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n); } /* If this is a new term, query for it. Update cksum3 with the results. */ @@ -4526,6 +4506,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT; fts5BufferFree(&term); + fts5BufferFree(&poslist); return fts5IndexReturn(p); } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 24a3521152..9c56790dc5 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -47,11 +47,9 @@ do_execsql_test 2.0 { do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); } -do_execsql_test 2.2 { - SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 -} { - {{structure idx=0} {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}} -} +do_test 2.2 { + execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } +} {/{{structure idx=0} {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/} do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('integrity-check'); } diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 23fdec0dfa..0746e64326 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -263,5 +263,27 @@ do_execsql_test 6.3 { SELECT rowid FROM s3 WHERE s3 MATCH 'a' } {1 2} +do_test 6.4 { + db close + sqlite3 db test.db + execsql { + BEGIN; + INSERT INTO s3(s3) VALUES('optimize'); + ROLLBACK; + } +} {} + +#------------------------------------------------------------------------- +# +set doc [string repeat "a b c " 500] +breakpoint +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + INSERT INTO x1 VALUES($doc); +} + + + finish_test diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index a5f657b160..74591cda78 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -28,7 +28,6 @@ do_execsql_test 1.0 { WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) INSERT INTO t1 SELECT rnddoc(10) FROM ii; } - set mask [expr 31 << 31] # Test 1: @@ -51,7 +50,6 @@ foreach {tno stmt} { 1 { DELETE FROM t1_data WHERE rowid=$rowid } 2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid } } { - break set tn 0 foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] { incr tn @@ -77,6 +75,8 @@ foreach {tno stmt} { } } +# Using the same database as the 1.* tests. +# # Run N-1 tests, where N is the number of bytes in the rightmost leaf page # of the fts index. For test $i, truncate the rightmost leafpage to $i # bytes. Then test both the integrity-check detects the corruption. @@ -112,5 +112,87 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { } {} } +#------------------------------------------------------------------------- +# Test that corruption in leaf page headers is detected by queries that use +# doclist-indexes. +# +set doc "A B C D E F G H I J " +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x3 USING fts5(tt); + INSERT INTO x3(x3, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000) + INSERT INTO x3 + SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii; +} + +foreach {tn hdr} { + 1 "\00\00\00\00" + 2 "\FF\FF\FF\FF" +} { + set tn2 0 + set nCorrupt 0 + foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] { + if {$rowid & $mask} continue + incr tn2 + do_test 3.$tn.$tn2 { + execsql BEGIN + + set fd [db incrblob main x3_data block $rowid] + fconfigure $fd -encoding binary -translation binary + puts -nonewline $fd $hdr + close $fd + + set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}] + if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + set {} 1 + } {1} + + execsql ROLLBACK + } + + do_test 3.$tn.x { expr $nCorrupt>0 } 1 +} + +#-------------------------------------------------------------------- +# +set doc "A B C D E F G H I J " +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x4 USING fts5(tt); + INSERT INTO x4(x4, rank) VALUES('pgsz', 32); + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) + INSERT INTO x4 + SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii; +} + +foreach {tn nCut} { + 1 1 + 2 10 +} { + set tn2 0 + set nCorrupt 0 + foreach rowid [db eval {SELECT rowid FROM x4_data WHERE rowid>10}] { + if {$rowid & $mask} continue + incr tn2 + do_test 4.$tn.$tn2 { + execsql { + BEGIN; + UPDATE x4_data SET block = substr(block, 1, length(block)-$nCut) + WHERE id = $rowid; + } + + set res [catchsql { + SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC + }] + if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + set {} 1 + } {1} + + execsql ROLLBACK + } + + do_test 4.$tn.x { expr $nCorrupt>0 } 1 +} + + finish_test diff --git a/ext/fts5/test/fts5fault2.test b/ext/fts5/test/fts5fault2.test index 8404e2f6ad..2624b5a8e4 100644 --- a/ext/fts5/test/fts5fault2.test +++ b/ext/fts5/test/fts5fault2.test @@ -51,5 +51,28 @@ do_faultsim_test 1.2 -faults oom-* -prep { faultsim_test_result {0 {1000 900 800 700 600 500 400 300 200 100}} } +#------------------------------------------------------------------------- +# OOM within a query that accesses the in-memory hash table. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE "a b c" USING fts5(a, b, c); + INSERT INTO "a b c" VALUES('one two', 'x x x', 'three four'); + INSERT INTO "a b c" VALUES('nine ten', 'y y y', 'two two'); +} + +do_faultsim_test 2.1 -faults oom-trans* -prep { + execsql { + BEGIN; + INSERT INTO "a b c" VALUES('one one', 'z z z', 'nine ten'); + } +} -body { + execsql { SELECT rowid FROM "a b c" WHERE "a b c" MATCH 'one' } +} -test { + faultsim_test_result {0 {1 3}} + catchsql { ROLLBACK } +} + + finish_test diff --git a/ext/fts5/test/fts5full.test b/ext/fts5/test/fts5full.test new file mode 100644 index 0000000000..4563cedce5 --- /dev/null +++ b/ext/fts5/test/fts5full.test @@ -0,0 +1,37 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that SQLITE_FULL is returned if the FTS5 table cannot find a free +# segid to use. In practice this can only really happen when automerge and +# crisismerge are both disabled. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5full + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x8 USING fts5(i); + INSERT INTO x8(x8, rank) VALUES('automerge', 0); + INSERT INTO x8(x8, rank) VALUES('crisismerge', 100000); +} + +db func rnddoc fts5_rnddoc +do_test 1.1 { + list [catch { + for {set i 0} {$i < 2500} {incr i} { + execsql { INSERT INTO x8 VALUES( rnddoc(5) ); } + } + } msg] $msg +} {1 {database or disk is full}} + + +finish_test + diff --git a/manifest b/manifest index 338f93c639..1be9a25a1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2015-04-24T20:18:21.844 +C Add\stests\sfor\sfts5. +D 2015-04-25T18:56:48.351 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23 F ext/fts5/fts5_config.c 43fcf838d3a3390d1245e3d5e651fa5cc1df575b F ext/fts5/fts5_expr.c 05da381ab26031243266069302c6eb4094b2c5dd F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529 -F ext/fts5/fts5_index.c 39810b25a017f2626ac72b3e44afe9b534e5d5db +F ext/fts5/fts5_index.c c87369d11271847df9f033f0df148e7f004a88a2 F ext/fts5/fts5_storage.c b3a4cbbcd197fe587789398e51a631f92fc9196c F ext/fts5/fts5_tcl.c 10bf0eb678d34c1bfdcfaf653d2e6dd92afa8b38 F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b @@ -120,8 +120,8 @@ F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32 -F ext/fts5/test/fts5aa.test 91f22b3cc7b372a2903c828e907a1e52f1177b8a -F ext/fts5/test/fts5ab.test 5da2e92a8047860b9e22b6fd3990549639d631b1 +F ext/fts5/test/fts5aa.test 87b2e01084c3e2a960ae1500dd5f0729dac2166c +F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d F ext/fts5/test/fts5ae.test 9175201baf8c885fc1cbb2da11a0c61fd11224db @@ -136,12 +136,13 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517 F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727 -F ext/fts5/test/fts5corrupt2.test 3be48d8a30d30e3ae819f04e957c45d091bfbb85 +F ext/fts5/test/fts5corrupt2.test 494111fd4f2dab36499cf97718eaba1f7c11e9d0 F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e F ext/fts5/test/fts5fault1.test ed71717a479bef32d05f02d9c48691011d160d4d -F ext/fts5/test/fts5fault2.test 000ecebf28b8f2cd520f44c70962625ee11d65ac +F ext/fts5/test/fts5fault2.test f478fa94e39a6911189f9e052a3b93ab4cd275fa +F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e @@ -1304,7 +1305,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 86309961344f4076ddcf55d730d3600ec3b6e45c dc88fe7e641c52d27fba8c753cee590db87388c5 -R e0fc7874fca6c46f36b4c18c43a12035 +P 1c78d8920fb59da3cb97dd2eb09b3e08dfd14259 +R 6572bfe0dee7d9becde3a8bb0a8d33a2 U dan -Z 339a1949f9a5bc9b03407831c5b374c4 +Z e7bb2196ce75c57e787365bfa027743a diff --git a/manifest.uuid b/manifest.uuid index 0185da7076..14320ee5eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c78d8920fb59da3cb97dd2eb09b3e08dfd14259 \ No newline at end of file +e748651c940eae2389fe826cf5c25f1166a5e611 \ No newline at end of file