Add further tests and fixes for fts5.
FossilOrigin-Name: 5b295897153e9b26cd0d2e7ea112a4d461d0a665
This commit is contained in:
parent
4c2871bead
commit
0b520cc5f9
@ -345,6 +345,9 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p);
|
||||
|
||||
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
|
||||
|
||||
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
|
||||
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
|
||||
|
||||
/*
|
||||
** End of interface to code in fts5_index.c.
|
||||
**************************************************************************/
|
||||
|
@ -111,11 +111,14 @@ void sqlite3Fts5HashFree(Fts5Hash *pHash){
|
||||
void sqlite3Fts5HashClear(Fts5Hash *pHash){
|
||||
int i;
|
||||
for(i=0; i<pHash->nSlot; i++){
|
||||
if( pHash->aSlot[i] ){
|
||||
sqlite3_free(pHash->aSlot[i]);
|
||||
pHash->aSlot[i] = 0;
|
||||
Fts5HashEntry *pNext;
|
||||
Fts5HashEntry *pSlot;
|
||||
for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
|
||||
pNext = pSlot->pNext;
|
||||
sqlite3_free(pSlot);
|
||||
}
|
||||
}
|
||||
memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
|
||||
pHash->nEntry = 0;
|
||||
}
|
||||
|
||||
|
@ -604,6 +604,72 @@ static u16 fts5GetU16(const u8 *aIn){
|
||||
return ((u16)aIn[0] << 8) + aIn[1];
|
||||
}
|
||||
|
||||
/*
|
||||
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
|
||||
** Except, this version does handle the single byte case that the core
|
||||
** version depends on being handled before its function is called.
|
||||
*/
|
||||
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
|
||||
u32 a,b;
|
||||
|
||||
/* The 1-byte case. Overwhelmingly the most common. */
|
||||
a = *p;
|
||||
/* a: p0 (unmasked) */
|
||||
if (!(a&0x80))
|
||||
{
|
||||
/* Values between 0 and 127 */
|
||||
*v = a;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* The 2-byte case */
|
||||
p++;
|
||||
b = *p;
|
||||
/* b: p1 (unmasked) */
|
||||
if (!(b&0x80))
|
||||
{
|
||||
/* Values between 128 and 16383 */
|
||||
a &= 0x7f;
|
||||
a = a<<7;
|
||||
*v = a | b;
|
||||
return 2;
|
||||
}
|
||||
|
||||
/* The 3-byte case */
|
||||
p++;
|
||||
a = a<<14;
|
||||
a |= *p;
|
||||
/* a: p0<<14 | p2 (unmasked) */
|
||||
if (!(a&0x80))
|
||||
{
|
||||
/* Values between 16384 and 2097151 */
|
||||
a &= (0x7f<<14)|(0x7f);
|
||||
b &= 0x7f;
|
||||
b = b<<7;
|
||||
*v = a | b;
|
||||
return 3;
|
||||
}
|
||||
|
||||
/* A 32-bit varint is used to store size information in btrees.
|
||||
** Objects are rarely larger than 2MiB limit of a 3-byte varint.
|
||||
** A 3-byte varint is sufficient, for example, to record the size
|
||||
** of a 1048569-byte BLOB or string.
|
||||
**
|
||||
** We only unroll the first 1-, 2-, and 3- byte cases. The very
|
||||
** rare larger cases can be handled by the slower 64-bit varint
|
||||
** routine.
|
||||
*/
|
||||
{
|
||||
u64 v64;
|
||||
u8 n;
|
||||
p -= 2;
|
||||
n = sqlite3GetVarint(p, &v64);
|
||||
*v = (u32)v64;
|
||||
assert( n>3 && n<=9 );
|
||||
return n;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
** Allocate and return a buffer at least nByte bytes in size.
|
||||
**
|
||||
@ -918,8 +984,8 @@ static int fts5StructureDecode(
|
||||
|
||||
/* Read the total number of levels and segments from the start of the
|
||||
** structure record. */
|
||||
i += getVarint32(&pData[i], nLevel);
|
||||
i += getVarint32(&pData[i], nSegment);
|
||||
i += fts5GetVarint32(&pData[i], nLevel);
|
||||
i += fts5GetVarint32(&pData[i], nSegment);
|
||||
nByte = (
|
||||
sizeof(Fts5Structure) + /* Main structure */
|
||||
sizeof(Fts5StructureLevel) * (nLevel) /* aLevel[] array */
|
||||
@ -935,8 +1001,8 @@ static int fts5StructureDecode(
|
||||
int nTotal;
|
||||
int iSeg;
|
||||
|
||||
i += getVarint32(&pData[i], pLvl->nMerge);
|
||||
i += getVarint32(&pData[i], nTotal);
|
||||
i += fts5GetVarint32(&pData[i], pLvl->nMerge);
|
||||
i += fts5GetVarint32(&pData[i], nTotal);
|
||||
assert( nTotal>=pLvl->nMerge );
|
||||
pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc,
|
||||
nTotal * sizeof(Fts5StructureSegment)
|
||||
@ -945,10 +1011,10 @@ static int fts5StructureDecode(
|
||||
if( rc==SQLITE_OK ){
|
||||
pLvl->nSeg = nTotal;
|
||||
for(iSeg=0; iSeg<nTotal; iSeg++){
|
||||
i += getVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
|
||||
i += getVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight);
|
||||
i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
|
||||
i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
|
||||
i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
|
||||
i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight);
|
||||
i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
|
||||
i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
|
||||
}
|
||||
}else{
|
||||
fts5StructureRelease(pRet);
|
||||
@ -1144,6 +1210,7 @@ static void fts5StructurePromoteTo(
|
||||
|
||||
for(il=iPromote+1; il<pStruct->nLevel; il++){
|
||||
Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
|
||||
if( pLvl->nMerge ) return;
|
||||
for(is=pLvl->nSeg-1; is>=0; is--){
|
||||
int sz = fts5SegmentSize(&pLvl->aSeg[is]);
|
||||
if( sz>szPromote ) return;
|
||||
@ -1193,7 +1260,8 @@ static void fts5StructurePromote(
|
||||
/* Check for condition (a) */
|
||||
for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--);
|
||||
pTst = &pStruct->aLevel[iTst];
|
||||
if( iTst>=0 && pTst->nMerge==0 ){
|
||||
assert( pTst->nMerge==0 );
|
||||
if( iTst>=0 ){
|
||||
int i;
|
||||
int szMax = 0;
|
||||
for(i=0; i<pTst->nSeg; i++){
|
||||
@ -1208,31 +1276,13 @@ static void fts5StructurePromote(
|
||||
}
|
||||
}
|
||||
|
||||
/* Check for condition (b) */
|
||||
/* If condition (a) is not met, assume (b) is true. StructurePromoteTo()
|
||||
** is a no-op if it is not. */
|
||||
if( iPromote<0 ){
|
||||
Fts5StructureLevel *pTst;
|
||||
for(iTst=iLvl+1; iTst<pStruct->nLevel; iTst++){
|
||||
pTst = &pStruct->aLevel[iTst];
|
||||
if( pTst->nSeg ) break;
|
||||
}
|
||||
if( iTst<pStruct->nLevel && pTst->nMerge==0 ){
|
||||
Fts5StructureSegment *pSeg2 = &pTst->aSeg[pTst->nSeg-1];
|
||||
int sz = pSeg2->pgnoLast - pSeg2->pgnoFirst + 1;
|
||||
if( sz<=szSeg ){
|
||||
iPromote = iLvl;
|
||||
szPromote = szSeg;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* If iPromote is greater than or equal to zero at this point, then it
|
||||
** is the level number of a level to which segments that consist of
|
||||
** szPromote or fewer pages should be promoted. */
|
||||
if( iPromote>=0 ){
|
||||
fts5PrintStructure("BEFORE", pStruct);
|
||||
fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
|
||||
fts5PrintStructure("AFTER", pStruct);
|
||||
iPromote = iLvl;
|
||||
szPromote = szSeg;
|
||||
}
|
||||
fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1246,7 +1296,7 @@ static void fts5NodeIterGobbleNEmpty(Fts5NodeIter *pIter){
|
||||
if( pIter->iOff<pIter->nData && 0==(pIter->aData[pIter->iOff] & 0xfe) ){
|
||||
pIter->bDlidx = pIter->aData[pIter->iOff] & 0x01;
|
||||
pIter->iOff++;
|
||||
pIter->iOff += getVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty);
|
||||
pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty);
|
||||
}else{
|
||||
pIter->nEmpty = 0;
|
||||
pIter->bDlidx = 0;
|
||||
@ -1262,8 +1312,8 @@ static void fts5NodeIterNext(int *pRc, Fts5NodeIter *pIter){
|
||||
pIter->iChild += pIter->nEmpty;
|
||||
}else{
|
||||
int nPre, nNew;
|
||||
pIter->iOff += getVarint32(&pIter->aData[pIter->iOff], nPre);
|
||||
pIter->iOff += getVarint32(&pIter->aData[pIter->iOff], nNew);
|
||||
pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nPre);
|
||||
pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nNew);
|
||||
pIter->term.n = nPre-2;
|
||||
fts5BufferAppendBlob(pRc, &pIter->term, nNew, pIter->aData+pIter->iOff);
|
||||
pIter->iOff += nNew;
|
||||
@ -1282,7 +1332,7 @@ static void fts5NodeIterInit(const u8 *aData, int nData, Fts5NodeIter *pIter){
|
||||
memset(pIter, 0, sizeof(*pIter));
|
||||
pIter->aData = aData;
|
||||
pIter->nData = nData;
|
||||
pIter->iOff = getVarint32(aData, pIter->iChild);
|
||||
pIter->iOff = fts5GetVarint32(aData, pIter->iChild);
|
||||
fts5NodeIterGobbleNEmpty(pIter);
|
||||
}
|
||||
|
||||
@ -1466,7 +1516,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
|
||||
int iOff = pIter->iLeafOffset; /* Offset to read at */
|
||||
int nNew; /* Bytes of new data */
|
||||
|
||||
iOff += getVarint32(&a[iOff], nNew);
|
||||
iOff += fts5GetVarint32(&a[iOff], nNew);
|
||||
pIter->term.n = nKeep;
|
||||
fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
|
||||
iOff += nNew;
|
||||
@ -1548,7 +1598,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
|
||||
i64 iDelta = 0;
|
||||
int nPos;
|
||||
|
||||
i += getVarint32(&a[i], nPos);
|
||||
i += fts5GetVarint32(&a[i], nPos);
|
||||
i += nPos;
|
||||
if( i>=n ) break;
|
||||
i += getVarint(&a[i], (u64*)&iDelta);
|
||||
@ -1665,7 +1715,7 @@ static void fts5SegIterNext(
|
||||
pIter->iRowidOffset--;
|
||||
|
||||
pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
|
||||
iOff += getVarint32(&a[iOff], nPos);
|
||||
iOff += fts5GetVarint32(&a[iOff], nPos);
|
||||
iOff += nPos;
|
||||
getVarint(&a[iOff], (u64*)&iDelta);
|
||||
pIter->iRowid += iDelta;
|
||||
@ -1685,7 +1735,7 @@ static void fts5SegIterNext(
|
||||
iOff = pIter->iLeafOffset;
|
||||
if( iOff<n ){
|
||||
int nPoslist;
|
||||
iOff += getVarint32(&a[iOff], nPoslist);
|
||||
iOff += fts5GetVarint32(&a[iOff], nPoslist);
|
||||
iOff += nPoslist;
|
||||
}
|
||||
|
||||
@ -1700,7 +1750,7 @@ static void fts5SegIterNext(
|
||||
fts5SegIterNextPage(p, pIter);
|
||||
pIter->iLeafOffset = 4;
|
||||
}else if( iOff!=fts5GetU16(&a[2]) ){
|
||||
pIter->iLeafOffset += getVarint32(&a[iOff], nKeep);
|
||||
pIter->iLeafOffset += fts5GetVarint32(&a[iOff], nKeep);
|
||||
}
|
||||
}else{
|
||||
pIter->iRowid -= iDelta;
|
||||
@ -1760,7 +1810,7 @@ static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
|
||||
i64 iDelta;
|
||||
|
||||
/* Position list size in bytes */
|
||||
iOff += getVarint32(&pLeaf->p[iOff], nPos);
|
||||
iOff += fts5GetVarint32(&pLeaf->p[iOff], nPos);
|
||||
iOff += nPos;
|
||||
if( iOff>=pLeaf->n ) break;
|
||||
|
||||
@ -1843,7 +1893,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
|
||||
int nPoslist;
|
||||
|
||||
/* iOff is currently the offset of the size field of a position list. */
|
||||
iOff += getVarint32(&pLeaf->p[iOff], nPoslist);
|
||||
iOff += fts5GetVarint32(&pLeaf->p[iOff], nPoslist);
|
||||
iOff += nPoslist;
|
||||
|
||||
if( iOff<pLeaf->n ){
|
||||
@ -2353,7 +2403,7 @@ static void fts5ChunkIterInit(
|
||||
pLeaf = pIter->pLeaf;
|
||||
}
|
||||
|
||||
iOff += getVarint32(&pLeaf->p[iOff], pIter->nRem);
|
||||
iOff += fts5GetVarint32(&pLeaf->p[iOff], pIter->nRem);
|
||||
pIter->n = MIN(pLeaf->n - iOff, pIter->nRem);
|
||||
pIter->p = pLeaf->p + iOff;
|
||||
|
||||
@ -2383,7 +2433,7 @@ static int fts5PosIterReadVarint(Fts5Index *p, Fts5PosIter *pIter){
|
||||
if( fts5ChunkIterEof(p, &pIter->chunk) ) return 0;
|
||||
pIter->iOff = 0;
|
||||
}
|
||||
pIter->iOff += getVarint32(&pIter->chunk.p[pIter->iOff], iVal);
|
||||
pIter->iOff += fts5GetVarint32(&pIter->chunk.p[pIter->iOff], iVal);
|
||||
}
|
||||
return iVal;
|
||||
}
|
||||
@ -2546,20 +2596,25 @@ static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
}
|
||||
|
||||
static void fts5WriteBtreeGrow(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
Fts5PageWriter *aNew;
|
||||
Fts5PageWriter *pNew;
|
||||
int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1);
|
||||
if( p->rc==SQLITE_OK ){
|
||||
Fts5PageWriter *aNew;
|
||||
Fts5PageWriter *pNew;
|
||||
int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1);
|
||||
|
||||
aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew);
|
||||
if( aNew==0 ) return;
|
||||
aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew);
|
||||
if( aNew==0 ){
|
||||
p->rc = SQLITE_NOMEM;
|
||||
return;
|
||||
}
|
||||
|
||||
pNew = &aNew[pWriter->nWriter];
|
||||
memset(pNew, 0, sizeof(Fts5PageWriter));
|
||||
pNew->pgno = 1;
|
||||
fts5BufferAppendVarint(&p->rc, &pNew->buf, 1);
|
||||
pNew = &aNew[pWriter->nWriter];
|
||||
memset(pNew, 0, sizeof(Fts5PageWriter));
|
||||
pNew->pgno = 1;
|
||||
fts5BufferAppendVarint(&p->rc, &pNew->buf, 1);
|
||||
|
||||
pWriter->nWriter++;
|
||||
pWriter->aWriter = aNew;
|
||||
pWriter->nWriter++;
|
||||
pWriter->aWriter = aNew;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@ -3150,13 +3205,15 @@ static void fts5IndexWork(
|
||||
#endif
|
||||
|
||||
if( nBest<p->pConfig->nAutomerge
|
||||
&& pStruct->aLevel[iBestLvl].nMerge==0
|
||||
){
|
||||
&& pStruct->aLevel[iBestLvl].nMerge==0
|
||||
){
|
||||
break;
|
||||
}
|
||||
fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
|
||||
fts5StructurePromote(p, iBestLvl+1, pStruct);
|
||||
assert( nRem==0 || p->rc==SQLITE_OK );
|
||||
if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
|
||||
fts5StructurePromote(p, iBestLvl+1, pStruct);
|
||||
}
|
||||
*ppStruct = pStruct;
|
||||
}
|
||||
}
|
||||
@ -3272,6 +3329,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
|
||||
pSeg->pgnoFirst = 1;
|
||||
pSeg->pgnoLast = pgnoLast;
|
||||
}
|
||||
fts5StructurePromote(p, 0, pStruct);
|
||||
}
|
||||
|
||||
if( p->pConfig->nAutomerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
|
||||
@ -3543,7 +3601,7 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
}else{
|
||||
int nTerm; /* Size of term on leaf in bytes */
|
||||
int res; /* Comparison of term and split-key */
|
||||
iOff += getVarint32(&pLeaf->p[iOff], nTerm);
|
||||
iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
|
||||
res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
|
||||
if( res==0 ) res = nTerm - iter.term.n;
|
||||
if( res<0 ){
|
||||
@ -3667,7 +3725,7 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
|
||||
}else{
|
||||
pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
|
||||
}
|
||||
pIter->i += getVarint32(&pIter->a[pIter->i], pIter->nPoslist);
|
||||
pIter->i += fts5GetVarint32(&pIter->a[pIter->i], pIter->nPoslist);
|
||||
pIter->aPoslist = &pIter->a[pIter->i];
|
||||
pIter->i += pIter->nPoslist;
|
||||
}else{
|
||||
@ -3829,7 +3887,7 @@ static void fts5SetupPrefixIter(
|
||||
&& ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid))
|
||||
){
|
||||
|
||||
for(i=0; doclist.n && p->rc==SQLITE_OK; i++){
|
||||
for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
|
||||
assert( i<nBuf );
|
||||
if( aBuf[i].n==0 ){
|
||||
fts5BufferSwap(&doclist, &aBuf[i]);
|
||||
@ -4499,7 +4557,7 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
|
||||
int iOff = 0;
|
||||
while( iOff<n ){
|
||||
int iVal;
|
||||
iOff += getVarint32(&a[iOff], iVal);
|
||||
iOff += fts5GetVarint32(&a[iOff], iVal);
|
||||
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
|
||||
}
|
||||
return iOff;
|
||||
@ -4523,7 +4581,7 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
|
||||
}
|
||||
while( iOff<n ){
|
||||
int nPos;
|
||||
iOff += getVarint32(&a[iOff], nPos);
|
||||
iOff += fts5GetVarint32(&a[iOff], nPos);
|
||||
iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
|
||||
if( iOff<n ){
|
||||
i64 iDelta;
|
||||
@ -4628,7 +4686,7 @@ static void fts5DecodeFunction(
|
||||
assert( iTermOff==0 || iOff==iTermOff );
|
||||
while( iOff<n ){
|
||||
int nByte;
|
||||
iOff += getVarint32(&a[iOff], nByte);
|
||||
iOff += fts5GetVarint32(&a[iOff], nByte);
|
||||
term.n= nKeep;
|
||||
fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
|
||||
iOff += nByte;
|
||||
@ -4638,7 +4696,7 @@ static void fts5DecodeFunction(
|
||||
);
|
||||
iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
|
||||
if( iOff<n ){
|
||||
iOff += getVarint32(&a[iOff], nKeep);
|
||||
iOff += fts5GetVarint32(&a[iOff], nKeep);
|
||||
}
|
||||
}
|
||||
fts5BufferFree(&term);
|
||||
|
@ -114,4 +114,21 @@ proc fts5_aux_test_functions {db} {
|
||||
}
|
||||
}
|
||||
|
||||
proc fts5_level_segs {tbl} {
|
||||
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
|
||||
set ret [list]
|
||||
foreach L [lrange [db one $sql] 1 end] {
|
||||
lappend ret [expr [llength $L] - 2]
|
||||
}
|
||||
set ret
|
||||
}
|
||||
|
||||
proc fts5_rnddoc {n} {
|
||||
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
|
||||
set doc [list]
|
||||
for {set i 0} {$i < $n} {incr i} {
|
||||
lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
|
||||
}
|
||||
set doc
|
||||
}
|
||||
|
||||
|
@ -149,5 +149,90 @@ foreach {tn expr res} {
|
||||
} $res
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Documents with more than 2M tokens.
|
||||
#
|
||||
|
||||
do_execsql_test 4.0 {
|
||||
CREATE VIRTUAL TABLE s1 USING fts5(x);
|
||||
}
|
||||
foreach {tn doc} [list \
|
||||
1 [string repeat {a x } 1500000] \
|
||||
2 "[string repeat {a a } 1500000] x" \
|
||||
] {
|
||||
do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) }
|
||||
}
|
||||
|
||||
do_execsql_test 4.3 {
|
||||
SELECT rowid FROM s1 WHERE s1 MATCH 'x'
|
||||
} {2 1}
|
||||
|
||||
do_execsql_test 4.4 {
|
||||
SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
|
||||
} {2 1}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Check that a special case of segment promotion works. The case is where
|
||||
# a new segment is written to level L, but the oldest segment within level
|
||||
# (L-2) is larger than it.
|
||||
#
|
||||
do_execsql_test 5.0 {
|
||||
CREATE VIRTUAL TABLE s2 USING fts5(x);
|
||||
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
|
||||
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
|
||||
}
|
||||
|
||||
proc rnddoc {n} {
|
||||
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
|
||||
set doc [list]
|
||||
for {set i 0} {$i < $n} {incr i} {
|
||||
lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]]
|
||||
}
|
||||
set doc
|
||||
}
|
||||
db func rnddoc rnddoc
|
||||
|
||||
do_test 5.1 {
|
||||
for {set i 1} {$i <= 65} {incr i} {
|
||||
execsql { INSERT INTO s2 VALUES(rnddoc(10)) }
|
||||
}
|
||||
for {set i 1} {$i <= 63} {incr i} {
|
||||
execsql { DELETE FROM s2 WHERE rowid = $i }
|
||||
}
|
||||
fts5_level_segs s2
|
||||
} {0 8}
|
||||
|
||||
do_test 5.2 {
|
||||
execsql {
|
||||
INSERT INTO s2(s2, rank) VALUES('automerge', 8);
|
||||
}
|
||||
for {set i 0} {$i < 7} {incr i} {
|
||||
execsql { INSERT INTO s2 VALUES(rnddoc(50)) }
|
||||
}
|
||||
fts5_level_segs s2
|
||||
} {8 0 0}
|
||||
|
||||
# Test also the other type of segment promotion - when a new segment is written
|
||||
# that is larger than segments immediately following it.
|
||||
do_test 5.3 {
|
||||
execsql {
|
||||
DROP TABLE s2;
|
||||
CREATE VIRTUAL TABLE s2 USING fts5(x);
|
||||
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
|
||||
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
|
||||
}
|
||||
|
||||
for {set i 1} {$i <= 16} {incr i} {
|
||||
execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
|
||||
}
|
||||
fts5_level_segs s2
|
||||
} {0 1}
|
||||
|
||||
do_test 5.4 {
|
||||
execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
|
||||
fts5_level_segs s2
|
||||
} {2 0}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
|
@ -210,8 +210,6 @@ do_faultsim_test 5.5.4 -faults oom* -body {
|
||||
faultsim_test_result [list 0 1]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
reset_db
|
||||
@ -276,8 +274,8 @@ do_faultsim_test 6.4 -faults oom-* -prep {
|
||||
faultsim_test_result [list 0 {}]
|
||||
}
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
do_faultsim_test 7.0 -faults oom* -prep {
|
||||
catch { db close }
|
||||
} -body {
|
||||
@ -286,6 +284,74 @@ do_faultsim_test 7.0 -faults oom* -prep {
|
||||
faultsim_test_result [list 0 {}] [list 1 {}]
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# A prefix query against a large document set.
|
||||
#
|
||||
proc rnddoc {n} {
|
||||
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
|
||||
set doc [list]
|
||||
for {set i 0} {$i < $n} {incr i} {
|
||||
lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
|
||||
}
|
||||
set doc
|
||||
}
|
||||
|
||||
reset_db
|
||||
db func rnddoc rnddoc
|
||||
|
||||
do_test 8.0 {
|
||||
execsql { CREATE VIRTUAL TABLE x1 USING fts5(a) }
|
||||
set ::res [list]
|
||||
for {set i 100} {$i>0} {incr i -1} {
|
||||
execsql { INSERT INTO x1 VALUES( rnddoc(50) ) }
|
||||
lappend ::res $i
|
||||
}
|
||||
} {}
|
||||
|
||||
do_faultsim_test 8.1 -faults oom* -prep {
|
||||
} -body {
|
||||
execsql {
|
||||
SELECT rowid FROM x1 WHERE x1 MATCH 'x*'
|
||||
}
|
||||
} -test {
|
||||
faultsim_test_result [list 0 $::res]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Segment promotion.
|
||||
#
|
||||
do_test 9.0 {
|
||||
reset_db
|
||||
db func rnddoc fts5_rnddoc
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE s2 USING fts5(x);
|
||||
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
|
||||
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
|
||||
}
|
||||
|
||||
for {set i 1} {$i <= 16} {incr i} {
|
||||
execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
|
||||
}
|
||||
fts5_level_segs s2
|
||||
} {0 1}
|
||||
faultsim_save_and_close
|
||||
|
||||
do_faultsim_test 9.1 -faults oom-* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
db func rnddoc fts5_rnddoc
|
||||
} -body {
|
||||
execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
if {$testrc==0} {
|
||||
set ls [fts5_level_segs s2]
|
||||
if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
|
22
manifest
22
manifest
@ -1,5 +1,5 @@
|
||||
C Further\stests\sand\sfixes\sfor\sfts5.
|
||||
D 2015-01-21T20:30:14.830
|
||||
C Add\sfurther\stests\sand\sfixes\sfor\sfts5.
|
||||
D 2015-01-22T19:13:08.439
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -106,22 +106,22 @@ F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e
|
||||
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
|
||||
F ext/fts5/fts5.c 0ba5a8f27e1aa4deab82f0fc295d55f67dfe7f34
|
||||
F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a
|
||||
F ext/fts5/fts5Int.h b593d5ff5f0cc6493778f88bc19db1dea42e003b
|
||||
F ext/fts5/fts5Int.h 99da8551098bb23fd94d0aa3f4ae1a411ee630b4
|
||||
F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
|
||||
F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9
|
||||
F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399
|
||||
F ext/fts5/fts5_expr.c 8a0e643768666dc2bffe74104141274809699808
|
||||
F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
|
||||
F ext/fts5/fts5_index.c baf26bfee5bd776194c0e508e3c101964ae851c6
|
||||
F ext/fts5/fts5_hash.c 7a87f9f2eae2216c710064821fa0621ac6a8ce7b
|
||||
F ext/fts5/fts5_index.c ee7b141adde3dbdaa56f1e198c06a0786d298126
|
||||
F ext/fts5/fts5_storage.c d56722960982d0c48ba1b88d9001fefed8cff1a4
|
||||
F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5
|
||||
F ext/fts5/fts5_tokenize.c 7c61d5c35c3449597bdeaa54dd48afe26852c7b0
|
||||
F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9
|
||||
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
|
||||
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
|
||||
F ext/fts5/test/fts5_common.tcl 08e939096a07eb77a7a986613e960f31d3cab2cc
|
||||
F ext/fts5/test/fts5_common.tcl 7db772d34fa0139d4b58d2b321928c9ccd30f699
|
||||
F ext/fts5/test/fts5aa.test 8ddbbcbedab67101dc9a86fd5c39d78b0e06515f
|
||||
F ext/fts5/test/fts5ab.test 91a3faac09ad9fab5f71494db6e4071963281536
|
||||
F ext/fts5/test/fts5ab.test 3f3ad2fb9ed60a0df57b626fa6fe6ef41d4deee0
|
||||
F ext/fts5/test/fts5ac.test 48181b7c873da0e3b4a3316760fcb90d88e7fbd8
|
||||
F ext/fts5/test/fts5ad.test 3b01eec8516d5631909716514e2e585a45ef0eb1
|
||||
F ext/fts5/test/fts5ae.test 014d5be2f5f70407fb032d4f27704116254797c3
|
||||
@ -136,7 +136,7 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
|
||||
F ext/fts5/test/fts5content.test 4234e0b11e003fe1e80472aa637f70464396fdd0
|
||||
F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
|
||||
F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
|
||||
F ext/fts5/test/fts5fault1.test 405886f4ecd255fa7a7077c021a65c9f8eaa4804
|
||||
F ext/fts5/test/fts5fault1.test f9bafb61b40061ad19b61d15003c5faeea4a57b5
|
||||
F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
|
||||
F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
|
||||
F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
|
||||
@ -1282,7 +1282,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
|
||||
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
||||
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
|
||||
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
|
||||
P e0d614425f7f5ffe266fdc03642931b1ec19ad25
|
||||
R c0a23f610b9ed738a5e0737db6938cae
|
||||
P c020a291ed293a66d21c5885e50a7fee04aa6366
|
||||
R 244beb886a9d1f5f10328b67a9ad3f5b
|
||||
U dan
|
||||
Z 2e4179b4673c6b65260289e1eab737a6
|
||||
Z 4d510e0e441ea9491ac2e8425faae5c2
|
||||
|
@ -1 +1 @@
|
||||
c020a291ed293a66d21c5885e50a7fee04aa6366
|
||||
5b295897153e9b26cd0d2e7ea112a4d461d0a665
|
Loading…
x
Reference in New Issue
Block a user