mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge recent changes from trunk.

Browse Source 
FossilOrigin-Name: 6eefdad946da6a5f4052ac51d327777890fa3f18
This commit is contained in:
drh 2014-05-19 23:17:33 +00:00
commit db2f91907d
13 changed files with 942 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ext/fts3/fts3.c
View File

@ -1333,7 +1333,7 @@ static int fts3InitVtab(
p->bHasStat = isFts4;
p->bFts4 = isFts4;
p->bDescIdx = bDescIdx;
p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */
p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */
p->zContentTbl = zContent;
p->zLanguageid = zLanguageid;
zContent = 0;
@ -3302,7 +3302,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
Fts3Table *p = (Fts3Table*)pVtab;
int rc = sqlite3Fts3PendingTermsFlush(p);
if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){
if( rc==SQLITE_OK
&& p->nLeafAdd>(nMinMerge/16)
&& p->nAutoincrmerge && p->nAutoincrmerge!=0xff
){
int mxLevel = 0; /* Maximum relative level value in db */
int A; /* Incr-merge parameter A */
@ -3310,7 +3313,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
assert( rc==SQLITE_OK || mxLevel==0 );
A = p->nLeafAdd * mxLevel;
A += (A/2);
if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8);
if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
}
sqlite3Fts3SegmentsClose(p);
return rc;

4
ext/fts3/fts3Int.h
View File

@ -210,13 +210,13 @@ struct Fts3Table {
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
char *zContentTbl; /* content=xxx option, or NULL */
char *zLanguageid; /* languageid=xxx option, or NULL */
u8 bAutoincrmerge; /* True if automerge=1 */
int nAutoincrmerge; /* Value configured by 'automerge' */
u32 nLeafAdd; /* Number of leaf blocks added this trans */
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
sqlite3_stmt *aStmt[37];
sqlite3_stmt *aStmt[40];
char *zReadExprlist;
char *zWriteExprlist;

305
ext/fts3/fts3_write.c
View File

@ -193,6 +193,7 @@ struct SegmentWriter {
int nSize; /* Size of allocation at aData */
int nData; /* Bytes of data in aData */
char *aData; /* Pointer to block from malloc() */
i64 nLeafData; /* Number of bytes of leaf data written */
};
/*
@ -268,6 +269,10 @@ struct SegmentNode {
#define SQL_SELECT_INDEXES 35
#define SQL_SELECT_MXLEVEL 36
#define SQL_SELECT_LEVEL_RANGE2 37
#define SQL_UPDATE_LEVEL_IDX 38
#define SQL_UPDATE_LEVEL 39
/*
** This function is used to obtain an SQLite prepared statement handle
** for the statement identified by the second argument. If successful,
@ -369,7 +374,18 @@ static int fts3SqlStmt(
/* SQL_SELECT_MXLEVEL
** Return the largest relative level in the FTS index or indexes. */
/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'"
/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'",
/* Return segments in order from oldest to newest.*/
/* 37 */ "SELECT level, idx, end_block "
"FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? "
"ORDER BY level DESC, idx ASC",
/* Update statements used while promoting segments */
/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? "
"WHERE level=? AND idx=?",
/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1"
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
@ -1910,6 +1926,7 @@ static int fts3WriteSegdir(
sqlite3_int64 iStartBlock, /* Value for "start_block" field */
sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
sqlite3_int64 iEndBlock, /* Value for "end_block" field */
sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */
char *zRoot, /* Blob value for "root" field */
int nRoot /* Number of bytes in buffer zRoot */
){
@ -1920,7 +1937,13 @@ static int fts3WriteSegdir(
sqlite3_bind_int(pStmt, 2, iIdx);
sqlite3_bind_int64(pStmt, 3, iStartBlock);
sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
sqlite3_bind_int64(pStmt, 5, iEndBlock);
if( nLeafData==0 ){
sqlite3_bind_int64(pStmt, 5, iEndBlock);
}else{
char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData);
if( !zEnd ) return SQLITE_NOMEM;
sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free);
}
sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
@ -2246,6 +2269,9 @@ static int fts3SegWriterAdd(
nDoclist; /* Doclist data */
}
/* Increase the total number of bytes written to account for the new entry. */
pWriter->nLeafData += nReq;
/* If the buffer currently allocated is too small for this entry, realloc
** the buffer to make it large enough.
*/
@ -2317,13 +2343,13 @@ static int fts3SegWriterFlush(
pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
}
if( rc==SQLITE_OK ){
rc = fts3WriteSegdir(
p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
rc = fts3WriteSegdir(p, iLevel, iIdx,
pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot);
}
}else{
/* The entire tree fits on the root node. Write it to the segdir table. */
rc = fts3WriteSegdir(
p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
rc = fts3WriteSegdir(p, iLevel, iIdx,
0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData);
}
p->nLeafAdd++;
return rc;
@ -2407,6 +2433,37 @@ static int fts3SegmentMaxLevel(
return sqlite3_reset(pStmt);
}
/*
** iAbsLevel is an absolute level that may be assumed to exist within
** the database. This function checks if it is the largest level number
** within its index. Assuming no error occurs, *pbMax is set to 1 if
** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK
** is returned. If an error occurs, an error code is returned and the
** final value of *pbMax is undefined.
*/
static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){
/* Set pStmt to the compiled version of:
**
** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
**
** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
*/
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pStmt, 1, iAbsLevel+1);
sqlite3_bind_int64(pStmt, 2,
((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL
);
*pbMax = 0;
if( SQLITE_ROW==sqlite3_step(pStmt) ){
*pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL;
}
return sqlite3_reset(pStmt);
}
/*
** Delete all entries in the %_segments table associated with the segment
** opened with seg-reader pSeg. This function does not affect the contents
@ -2942,6 +2999,140 @@ void sqlite3Fts3SegReaderFinish(
}
}
/*
** Decode the "end_block" field, selected by column iCol of the SELECT
** statement passed as the first argument.
**
** The "end_block" field may contain either an integer, or a text field
** containing the text representation of two non-negative integers separated
** by one or more space (0x20) characters. In the first case, set *piEndBlock
** to the integer value and *pnByte to zero before returning. In the second,
** set *piEndBlock to the first value and *pnByte to the second.
*/
static void fts3ReadEndBlockField(
sqlite3_stmt *pStmt,
int iCol,
i64 *piEndBlock,
i64 *pnByte
){
const unsigned char *zText = sqlite3_column_text(pStmt, iCol);
if( zText ){
int i;
int iMul = 1;
i64 iVal = 0;
for(i=0; zText[i]>='0' && zText[i]<='9'; i++){
iVal = iVal*10 + (zText[i] - '0');
}
*piEndBlock = iVal;
while( zText[i]==' ' ) i++;
iVal = 0;
if( zText[i]=='-' ){
i++;
iMul = -1;
}
for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){
iVal = iVal*10 + (zText[i] - '0');
}
*pnByte = (iVal * (i64)iMul);
}
}
/*
** A segment of size nByte bytes has just been written to absolute level
** iAbsLevel. Promote any segments that should be promoted as a result.
*/
static int fts3PromoteSegments(
Fts3Table *p, /* FTS table handle */
sqlite3_int64 iAbsLevel, /* Absolute level just updated */
sqlite3_int64 nByte /* Size of new segment at iAbsLevel */
){
int rc = SQLITE_OK;
sqlite3_stmt *pRange;
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0);
if( rc==SQLITE_OK ){
int bOk = 0;
i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1;
i64 nLimit = (nByte*3)/2;
/* Loop through all entries in the %_segdir table corresponding to
** segments in this index on levels greater than iAbsLevel. If there is
** at least one such segment, and it is possible to determine that all
** such segments are smaller than nLimit bytes in size, they will be
** promoted to level iAbsLevel. */
sqlite3_bind_int64(pRange, 1, iAbsLevel+1);
sqlite3_bind_int64(pRange, 2, iLast);
while( SQLITE_ROW==sqlite3_step(pRange) ){
i64 nSize, dummy;
fts3ReadEndBlockField(pRange, 2, &dummy, &nSize);
if( nSize<=0 || nSize>nLimit ){
/* If nSize==0, then the %_segdir.end_block field does not not
** contain a size value. This happens if it was written by an
** old version of FTS. In this case it is not possible to determine
** the size of the segment, and so segment promotion does not
** take place. */
bOk = 0;
break;
}
bOk = 1;
}
rc = sqlite3_reset(pRange);
if( bOk ){
int iIdx = 0;
sqlite3_stmt *pUpdate1;
sqlite3_stmt *pUpdate2;
if( rc==SQLITE_OK ){
rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0);
}
if( rc==SQLITE_OK ){
rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0);
}
if( rc==SQLITE_OK ){
/* Loop through all %_segdir entries for segments in this index with
** levels equal to or greater than iAbsLevel. As each entry is visited,
** updated it to set (level = -1) and (idx = N), where N is 0 for the
** oldest segment in the range, 1 for the next oldest, and so on.
**
** In other words, move all segments being promoted to level -1,
** setting the "idx" fields as appropriate to keep them in the same
** order. The contents of level -1 (which is never used, except
** transiently here), will be moved back to level iAbsLevel below. */
sqlite3_bind_int64(pRange, 1, iAbsLevel);
while( SQLITE_ROW==sqlite3_step(pRange) ){
sqlite3_bind_int(pUpdate1, 1, iIdx++);
sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0));
sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1));
sqlite3_step(pUpdate1);
rc = sqlite3_reset(pUpdate1);
if( rc!=SQLITE_OK ){
sqlite3_reset(pRange);
break;
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3_reset(pRange);
}
/* Move level -1 to level iAbsLevel */
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pUpdate2, 1, iAbsLevel);
sqlite3_step(pUpdate2);
rc = sqlite3_reset(pUpdate2);
}
}
}
return rc;
}
/*
** Merge all level iLevel segments in the database into a single
** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
@ -2966,6 +3157,7 @@ static int fts3SegmentMerge(
Fts3SegFilter filter; /* Segment term filter condition */
Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
int bIgnoreEmpty = 0; /* True to ignore empty segments */
i64 iMaxLevel = 0; /* Max level number for this index/langid */
assert( iLevel==FTS3_SEGCURSOR_ALL
|| iLevel==FTS3_SEGCURSOR_PENDING
@ -2977,6 +3169,11 @@ static int fts3SegmentMerge(
rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr);
if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel);
if( rc!=SQLITE_OK ) goto finished;
}
if( iLevel==FTS3_SEGCURSOR_ALL ){
/* This call is to merge all segments in the database to a single
** segment. The level of the new segment is equal to the numerically
@ -2986,21 +3183,21 @@ static int fts3SegmentMerge(
rc = SQLITE_DONE;
goto finished;
}
rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel);
iNewLevel = iMaxLevel;
bIgnoreEmpty = 1;
}else if( iLevel==FTS3_SEGCURSOR_PENDING ){
iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0);
rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx);
}else{
/* This call is to merge all segments at level iLevel. find the next
** available segment index at level iLevel+1. The call to
** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
** a single iLevel+2 segment if necessary. */
rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
assert( FTS3_SEGCURSOR_PENDING==-1 );
iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1);
rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel);
}
if( rc!=SQLITE_OK ) goto finished;
assert( csr.nSegment>0 );
assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
@ -3017,7 +3214,7 @@ static int fts3SegmentMerge(
csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
}
if( rc!=SQLITE_OK ) goto finished;
assert( pWriter );
assert( pWriter || bIgnoreEmpty );
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
rc = fts3DeleteSegdir(
@ -3025,7 +3222,14 @@ static int fts3SegmentMerge(
);
if( rc!=SQLITE_OK ) goto finished;
}
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
if( pWriter ){
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
if( rc==SQLITE_OK ){
if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevel<iMaxLevel ){
rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData);
}
}
}
finished:
fts3SegWriterFree(pWriter);
@ -3035,7 +3239,7 @@ static int fts3SegmentMerge(
/*
** Flush the contents of pendingTerms to level 0 segments.
** Flush the contents of pendingTerms to level 0 segments.
*/
int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
int rc = SQLITE_OK;
@ -3051,14 +3255,19 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
** estimate the number of leaf blocks of content to be written
*/
if( rc==SQLITE_OK && p->bHasStat
&& p->bAutoincrmerge==0xff && p->nLeafAdd>0
&& p->nAutoincrmerge==0xff && p->nLeafAdd>0
){
sqlite3_stmt *pStmt = 0;
rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
rc = sqlite3_step(pStmt);
p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
if( rc==SQLITE_ROW ){
p->nAutoincrmerge = sqlite3_column_int(pStmt, 0);
if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8;
}else if( rc==SQLITE_DONE ){
p->nAutoincrmerge = 0;
}
rc = sqlite3_reset(pStmt);
}
}
@ -3426,6 +3635,8 @@ struct IncrmergeWriter {
int iIdx; /* Index of *output* segment in iAbsLevel+1 */
sqlite3_int64 iStart; /* Block number of first allocated block */
sqlite3_int64 iEnd; /* Block number of last allocated block */
sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */
u8 bNoLeafData; /* If true, store 0 for segment size */
NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
};
@ -3764,8 +3975,8 @@ static int fts3IncrmergeAppend(
nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
}
pWriter->nLeafData += nSpace;
blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc);
if( rc==SQLITE_OK ){
if( pLeaf->block.n==0 ){
pLeaf->block.n = 1;
@ -3864,6 +4075,7 @@ static void fts3IncrmergeRelease(
pWriter->iStart, /* start_block */
pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */
pWriter->iEnd, /* end_block */
(pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */
pRoot->block.a, pRoot->block.n /* root */
);
}
@ -3965,7 +4177,11 @@ static int fts3IncrmergeLoad(
if( sqlite3_step(pSelect)==SQLITE_ROW ){
iStart = sqlite3_column_int64(pSelect, 1);
iLeafEnd = sqlite3_column_int64(pSelect, 2);
iEnd = sqlite3_column_int64(pSelect, 3);
fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
if( pWriter->nLeafData<0 ){
pWriter->nLeafData = pWriter->nLeafData * -1;
}
pWriter->bNoLeafData = (pWriter->nLeafData==0);
nRoot = sqlite3_column_bytes(pSelect, 4);
aRoot = sqlite3_column_blob(pSelect, 4);
}else{
@ -4566,11 +4782,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
/*
** Attempt an incremental merge that writes nMerge leaf blocks.
**
** Incremental merges happen nMin segments at a time. The two
** segments to be merged are the nMin oldest segments (the ones with
** the smallest indexes) in the highest level that contains at least
** nMin segments. Multiple merges might occur in an attempt to write the
** quota of nMerge leaf blocks.
** Incremental merges happen nMin segments at a time. The segments
** to be merged are the nMin oldest segments (the ones with the smallest
** values for the _segdir.idx field) in the highest level that contains
** at least nMin segments. Multiple merges might occur in an attempt to
** write the quota of nMerge leaf blocks.
*/
int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
int rc; /* Return code */
@ -4595,6 +4811,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;
sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
int bUseHint = 0; /* True if attempting to append */
int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
/* Search the %_segdir table for the absolute level with the smallest
** relative level number that contains at least nMin segments, if any.
@ -4648,6 +4865,19 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
** to start work on some other level. */
memset(pWriter, 0, nAlloc);
pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
if( rc==SQLITE_OK ){
rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
assert( bUseHint==1 || bUseHint==0 );
if( iIdx==0 || (bUseHint && iIdx==1) ){
int bIgnore;
rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore);
if( bIgnore ){
pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY;
}
}
}
if( rc==SQLITE_OK ){
rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
}
@ -4655,16 +4885,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
&& SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
&& SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
){
int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
if( rc==SQLITE_OK ){
if( bUseHint && iIdx>0 ){
const char *zKey = pCsr->zTerm;
int nKey = pCsr->nTerm;
rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
}else{
rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
}
if( bUseHint && iIdx>0 ){
const char *zKey = pCsr->zTerm;
int nKey = pCsr->nTerm;
rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
}else{
rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
}
if( rc==SQLITE_OK && pWriter->nLeafEst ){
@ -4686,7 +4912,13 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
}
}
if( nSeg!=0 ){
pWriter->nLeafData = pWriter->nLeafData * -1;
}
fts3IncrmergeRelease(p, pWriter, &rc);
if( nSeg==0 && pWriter->bNoLeafData==0 ){
fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
}
}
sqlite3Fts3SegReaderFinish(pCsr);
@ -4773,7 +5005,10 @@ static int fts3DoAutoincrmerge(
){
int rc = SQLITE_OK;
sqlite3_stmt *pStmt = 0;
p->bAutoincrmerge = fts3Getint(&zParam)!=0;
p->nAutoincrmerge = fts3Getint(&zParam);
if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
p->nAutoincrmerge = 8;
}
if( !p->bHasStat ){
assert( p->bFts4==0 );
sqlite3Fts3CreateStatTable(&rc, p);
@ -4782,7 +5017,7 @@ static int fts3DoAutoincrmerge(
rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
if( rc ) return rc;
sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
return rc;

30
manifest
View File

@ -1,5 +1,5 @@
C Rearrange\ssome\sconditionals\sand\sadd\s#if\sstatements\sto\smake\sthe\scode\smore\ntestable.
D 2014-05-16T20:24:51.024
C Merge\srecent\schanges\sfrom\strunk.
D 2014-05-19T23:17:33.708
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in de92112472618cb869d27249966bad1783e4a853
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db
F ext/fts3/fts3.c e83f894cf1adaf8decd6b1de76bfdcdb79b25507
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4
F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918
F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365
F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970
F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9
F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d
F ext/fts3/fts3_write.c 74c00329006c3ed6325ba4e5ab7c9b5fc99c8934
F ext/fts3/fts3_write.c 9e4e8579ad26cff3ee7743aaf5c3fe937fc441b4
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
@ -187,7 +187,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12
F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303
F src/main.c c2005c6386b087532757360b86584d0af5a4d02c
F src/main.c 14f02e450d8e5f78af7e75905632dd785cf93363
F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b
@ -337,7 +337,7 @@ F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61
F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74
F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85
F test/backcompat.test 5f8ad58b3eaebc78cd2c66c65476a42e6f32b2ad
F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4
F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62
F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf
F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4
@ -540,7 +540,7 @@ F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de
F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f
F test/fts3d.test 95c17d1b67b33a5eac0bf5a0d11116a0c0ac7a3a
F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963
F test/fts3defer2.test e880e3b65bdf999f4746cdaefa65f14a98b9b724
F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
@ -571,12 +571,14 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca
F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8
F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06
F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01
F test/fts4growth.test df10fde9f47cf5c71861e63fd8efcd573c4f7e53
F test/fts4growth2.test 2f063be1902a73cd087355837c52fed42ac11a5d
F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9
F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584
F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057
F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
@ -745,7 +747,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
F test/permutations.test 33e7e239ba494fdb30e2f4ffc64c508b145ff42f
F test/permutations.test 5da30f29e9bc59cf21c891ed1360b14d5d777c68
F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@ -1011,7 +1013,7 @@ F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1
F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
F test/tpch01.test 8f4ac52f62f3e9f6bce0889105aecdf0275e331b
F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5
F test/trace2.test e7a988fdd982cdec62f1f1f34b0360e6476d01a0
F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382
F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6
F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76
F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732
@ -1175,7 +1177,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P cceac14fd83ddd8f868c1767cdc66635607cb159
R 48ad2b172d9e67d362d3e27e91c64651
P 17afd77057f8695733a9a60225646c1d8813b1a0 8180e320ee4090e41511836678e49a98c0b228e8
R caf94a64ae09103b6f818ad8457d731e
U drh
Z 9751e875adf689a22af045a3a24a99e5
Z 654b19280087cc59a60d4e21f62a0c05

2
manifest.uuid
View File

@ -1 +1 @@
17afd77057f8695733a9a60225646c1d8813b1a0
6eefdad946da6a5f4052ac51d327777890fa3f18

7
src/main.c
View File

@ -3136,7 +3136,12 @@ int sqlite3_test_control(int op, ...){
** sqlite3_test_control().
*/
case SQLITE_TESTCTRL_FAULT_INSTALL: {
sqlite3Config.xTestCallback = va_arg(ap, int(*)(int));
/* MSVC is picky about pulling func ptrs from va lists.
** http://support.microsoft.com/kb/47961
** sqlite3Config.xTestCallback = va_arg(ap, int(*)(int));
*/
typedef int(*TESTCALLBACKFUNC_t)(int);
sqlite3Config.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t);
rc = sqlite3FaultSim(0);
break;
}

54
test/backcompat.test
View File

@ -58,12 +58,24 @@ proc do_backcompat_test {rv bin1 bin2 script} {
code1 { sqlite3 db test.db }
code2 { sqlite3 db test.db }
foreach c {code1 code2} {
$c {
set v [split [db version] .]
if {[llength $v]==3} {lappend v 0}
set ::sqlite_libversion [format \
"%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3]
]
}
}
uplevel $script
catch { code1 { db close } }
catch { code2 { db close } }
catch { close $::bc_chan2 }
catch { close $::bc_chan1 }
}
array set ::incompatible [list]
@ -381,6 +393,48 @@ ifcapable fts3 {
} {
do_test backcompat-3.7 [list sql1 $q] [sql2 $q]
}
# Now test that an incremental merge can be started by one version
# and finished by another. And that the integrity-check still
# passes.
do_test backcompat-3.8 {
sql1 {
DROP TABLE IF EXISTS t1;
DROP TABLE IF EXISTS t2;
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE t2 USING fts3(words);
}
code1 [list source $testdir/genesis.tcl]
code1 { fts_kjv_genesis }
sql1 {
INSERT INTO t2 SELECT words FROM t1;
INSERT INTO t2 SELECT words FROM t1;
INSERT INTO t2 SELECT words FROM t1;
INSERT INTO t2 SELECT words FROM t1;
INSERT INTO t2 SELECT words FROM t1;
INSERT INTO t2 SELECT words FROM t1;
SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level;
}
} {0 {0 1 2 3 4 5}}
if {[code1 { set ::sqlite_libversion }] >=3071200
&& [code2 { set ::sqlite_libversion }] >=3071200
} {
do_test backcompat-3.9 {
sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); }
sql2 {
SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level;
}
} {0 {0 1} 1 0}
do_test backcompat-3.10 {
sql1 { INSERT INTO t2(t2) VALUES('integrity-check') }
sql2 { INSERT INTO t2(t2) VALUES('integrity-check') }
} {}
}
}
}
}

19
test/fts3d.test
View File

@ -213,16 +213,17 @@ do_test fts3d-4.matches {
{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \
{0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}]
check_terms_all fts3d-4.1 {a four is one test that this three two was}
puts [db eval {SELECT c FROM t1 } ]
check_terms_all fts3d-4.1 {a four is test that this was}
check_doclist_all fts3d-4.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]}
check_doclist_all fts3d-4.1.2 four {}
check_doclist_all fts3d-4.1.3 is {[1 0[1]] [3 0[1]]}
check_doclist_all fts3d-4.1.4 one {}
#check_doclist_all fts3d-4.1.4 one {}
check_doclist_all fts3d-4.1.5 test {[1 0[3]] [2 0[3]] [3 0[3]]}
check_doclist_all fts3d-4.1.6 that {[2 0[0]]}
check_doclist_all fts3d-4.1.7 this {[1 0[0]] [3 0[0]]}
check_doclist_all fts3d-4.1.8 three {}
check_doclist_all fts3d-4.1.9 two {}
#check_doclist_all fts3d-4.1.8 three {}
#check_doclist_all fts3d-4.1.9 two {}
check_doclist_all fts3d-4.1.10 was {[2 0[1]]}
check_terms fts3d-4.2 0 0 {a four test that was}
@ -239,16 +240,16 @@ check_doclist fts3d-4.3.3 0 1 is {[3 0[1]]}
check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]}
check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]}
check_terms fts3d-4.4 1 0 {a four is one test that this three two was}
check_terms fts3d-4.4 1 0 {a four is test that this was}
check_doclist fts3d-4.4.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]}
check_doclist fts3d-4.4.2 1 0 four {[1] [2 0[4]] [3 0[4]]}
check_doclist fts3d-4.4.2 1 0 four {[2 0[4]] [3 0[4]]}
check_doclist fts3d-4.4.3 1 0 is {[1 0[1]] [3 0[1]]}
check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]}
#check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]}
check_doclist fts3d-4.4.5 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]}
check_doclist fts3d-4.4.6 1 0 that {[2 0[0]]}
check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]}
check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]}
check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]}
#check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]}
#check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]}
check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]}
# Optimize should leave the result in the level of the highest-level

437
test/fts4growth.test Normal file
View File

@ -0,0 +1,437 @@
# 2014 May 12
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS4 module.
#
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts4growth
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
source $testdir/genesis.tcl
do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; }
do_test 1.2 {
foreach L {
{"See here, young man," said Mulga Bill, "from Walgett to the sea,}
{From Conroy's Gap to Castlereagh, there's none can ride like me.}
{I'm good all round at everything as everybody knows,}
{Although I'm not the one to talk -- I hate a man that blows.}
} {
execsql { INSERT INTO x1 VALUES($L) }
}
execsql { SELECT end_block, length(root) FROM x1_segdir }
} {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115}
do_execsql_test 1.3 {
INSERT INTO x1(x1) VALUES('optimize');
SELECT level, end_block, length(root) FROM x1_segdir;
} {0 {0 394} 394}
do_test 1.4 {
foreach L {
{But riding is my special gift, my chiefest, sole delight;}
{Just ask a wild duck can it swim, a wildcat can it fight.}
{There's nothing clothed in hair or hide, or built of flesh or steel,}
{There's nothing walks or jumps, or runs, on axle, hoof, or wheel,}
{But what I'll sit, while hide will hold and girths and straps are tight:}
{I'll ride this here two-wheeled concern right straight away at sight."}
} {
execsql { INSERT INTO x1 VALUES($L) }
}
execsql {
INSERT INTO x1(x1) VALUES('merge=4,4');
SELECT level, end_block, length(root) FROM x1_segdir;
}
} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2}
do_execsql_test 1.5 {
SELECT length(block) FROM x1_segments;
} {658 {}}
do_test 1.6 {
foreach L {
{'Twas Mulga Bill, from Eaglehawk, that sought his own abode,}
{That perched above Dead Man's Creek, beside the mountain road.}
{He turned the cycle down the hill and mounted for the fray,}
{But 'ere he'd gone a dozen yards it bolted clean away.}
{It left the track, and through the trees, just like a silver steak,}
{It whistled down the awful slope towards the Dead Man's Creek.}
{It shaved a stump by half an inch, it dodged a big white-box:}
{The very wallaroos in fright went scrambling up the rocks,}
{The wombats hiding in their caves dug deeper underground,}
{As Mulga Bill, as white as chalk, sat tight to every bound.}
{It struck a stone and gave a spring that cleared a fallen tree,}
{It raced beside a precipice as close as close could be;}
{And then as Mulga Bill let out one last despairing shriek}
{It made a leap of twenty feet into the Dead Man's Creek.}
} {
execsql { INSERT INTO x1 VALUES($L) }
}
execsql {
SELECT level, end_block, length(root) FROM x1_segdir;
}
} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117}
do_execsql_test 1.7 {
SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130);
} {1377}
#-------------------------------------------------------------------------
#
do_execsql_test 2.1 {
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE x2 USING fts4;
}
fts_kjv_genesis
do_test 2.2 {
foreach id [db eval {SELECT docid FROM t1}] {
execsql {
INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id
}
}
foreach id [db eval {SELECT docid FROM t1}] {
execsql {
INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id
}
if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break
}
} {}
do_execsql_test 2.3 {
SELECT count(*) FROM x2_segdir WHERE level=2;
SELECT count(*) FROM x2_segdir WHERE level=3;
} {6 0}
do_execsql_test 2.4 {
INSERT INTO x2(x2) VALUES('merge=4,4');
SELECT count(*) FROM x2_segdir WHERE level=2;
SELECT count(*) FROM x2_segdir WHERE level=3;
} {6 1}
do_execsql_test 2.5 {
SELECT end_block FROM x2_segdir WHERE level=3;
INSERT INTO x2(x2) VALUES('merge=4,4');
SELECT end_block FROM x2_segdir WHERE level=3;
INSERT INTO x2(x2) VALUES('merge=4,4');
SELECT end_block FROM x2_segdir WHERE level=3;
} {{3828 -3430} {3828 -10191} {3828 -14109}}
do_execsql_test 2.6 {
SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE
blockid BETWEEN start_block AND leaves_end_block
AND level=3
} {14109}
do_execsql_test 2.7 {
INSERT INTO x2(x2) VALUES('merge=1000,4');
SELECT end_block FROM x2_segdir WHERE level=3;
} {{3828 86120}}
do_execsql_test 2.8 {
SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE
blockid BETWEEN start_block AND leaves_end_block
AND level=3
} {86120}
#--------------------------------------------------------------------------
# Test that delete markers are removed from FTS segments when possible.
# It is only possible to remove delete markers when the output of the
# merge operation will become the oldest segment in the index.
#
# 3.1 - when the oldest segment is created by an 'optimize'.
# 3.2 - when the oldest segment is created by an incremental merge.
# 3.3 - by a crisis merge.
#
proc insert_doc {args} {
foreach iDoc $args {
set L [lindex {
{In your eagerness to engage the Trojans,}
{don’t any of you charge ahead of others,}
{trusting in your strength and horsemanship.}
{And don’t lag behind. That will hurt our charge.}
{Any man whose chariot confronts an enemy’s}
{should thrust with his spear at him from there.}
{That’s the most effective tactic, the way}
{men wiped out city strongholds long ago —}
{their chests full of that style and spirit.}
} [expr $iDoc%9]]
execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) }
}
}
proc delete_doc {args} {
foreach iDoc $args {
execsql { DELETE FROM x3 WHERE docid = $iDoc }
}
}
proc second {x} { lindex $x 1 }
db func second second
do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 }
do_test 3.1.1 {
db transaction { insert_doc 1 2 3 4 5 6 }
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {0 0 412}
do_test 3.1.2 {
delete_doc 1 2 3 4 5 6
execsql { SELECT count(*) FROM x3_segdir }
} {0}
do_test 3.1.3 {
db transaction {
insert_doc 1 2 3 4 5 6 7 8 9
delete_doc 9 8 7
}
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {0 0 591 0 1 65 0 2 72 0 3 76}
do_test 3.1.4 {
execsql { INSERT INTO x3(x3) VALUES('optimize') }
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {0 0 412}
do_test 3.2.1 {
execsql { DELETE FROM x3 }
insert_doc 8 7 6 5 4 3 2 1
delete_doc 7 8
execsql { SELECT count(*) FROM x3_segdir }
} {10}
do_test 3.2.2 {
execsql { INSERT INTO x3(x3) VALUES('merge=500,10') }
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {1 0 412}
# This assumes the crisis merge happens when there are already 16
# segments and one more is added.
#
do_test 3.3.1 {
execsql { DELETE FROM x3 }
insert_doc 1 2 3 4 5 6 7 8 9 10 11
delete_doc 11 10 9 8 7
execsql { SELECT count(*) FROM x3_segdir }
} {16}
do_test 3.3.2 {
insert_doc 12
execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 }
} {1 0 412}
#--------------------------------------------------------------------------
# Check a theory on a bug in fts4 - that segments with idx==0 were not
# being incrementally merged correctly. Theory turned out to be false.
#
do_execsql_test 4.1 {
DROP TABLE IF EXISTS x4;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE x4 USING fts4(words);
}
do_test 4.2 {
fts_kjv_genesis
execsql { INSERT INTO x4 SELECT words FROM t1 }
execsql { INSERT INTO x4 SELECT words FROM t1 }
} {}
do_execsql_test 4.3 {
SELECT level, idx, second(end_block) FROM x4_segdir
} {0 0 117483 0 1 118006}
do_execsql_test 4.4 {
INSERT INTO x4(x4) VALUES('merge=10,2');
SELECT count(*) FROM x4_segdir;
} {3}
do_execsql_test 4.5 {
INSERT INTO x4(x4) VALUES('merge=10,2');
SELECT count(*) FROM x4_segdir;
} {3}
do_execsql_test 4.6 {
INSERT INTO x4(x4) VALUES('merge=1000,2');
SELECT count(*) FROM x4_segdir;
} {1}
#--------------------------------------------------------------------------
# Check that segments are not promoted if the "end_block" field does not
# contain a size.
#
do_execsql_test 5.1 {
DROP TABLE IF EXISTS x2;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE x2 USING fts4;
}
fts_kjv_genesis
proc first {L} {lindex $L 0}
db func first first
do_test 5.2 {
foreach r [db eval { SELECT rowid FROM t1 }] {
execsql {
INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r
}
}
foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] {
execsql { DELETE FROM x2 WHERE docid = $d }
}
execsql {
INSERT INTO x2(x2) VALUES('optimize');
SELECT level, idx, end_block FROM x2_segdir
}
} {2 0 {752 1926}}
do_execsql_test 5.3 {
UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER );
SELECT end_block, typeof(end_block) FROM x2_segdir;
} {752 integer}
do_execsql_test 5.4 {
INSERT INTO x2 SELECT words FROM t1 LIMIT 50;
SELECT level, idx, end_block FROM x2_segdir
} {2 0 752 0 0 {758 5174}}
do_execsql_test 5.5 {
UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2;
INSERT INTO x2 SELECT words FROM t1 LIMIT 40;
SELECT level, idx, end_block FROM x2_segdir
} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}}
proc t1_to_x2 {} {
foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] {
execsql {
DELETE FROM x2 WHERE docid=$id;
INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id;
}
}
}
#--------------------------------------------------------------------------
# Check that segments created by auto-merge are not promoted until they
# are completed.
#
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE x5 USING fts4;
INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0;
INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25;
INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50;
INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75;
SELECT count(*) FROM x5_segdir
} {4}
do_execsql_test 6.2 {
INSERT INTO x5(x5) VALUES('merge=2,4');
SELECT level, idx, end_block FROM x5_segdir;
} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}}
do_execsql_test 6.3 {
INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100;
SELECT level, idx, end_block FROM x5_segdir;
} {
0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850}
0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297}
}
do_execsql_test 6.4 {
INSERT INTO x5(x5) VALUES('merge=200,4');
SELECT level, idx, end_block FROM x5_segdir;
} {0 0 {1329 8297} 1 0 {1320 28009}}
do_execsql_test 6.5 {
INSERT INTO x5 SELECT words FROM t1;
SELECT level, idx, end_block FROM x5_segdir;
} {
0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
}
#--------------------------------------------------------------------------
# Ensure that if part of an incremental merge is performed by an old
# version that does not support storing segment sizes in the end_block
# field, no size is stored in the final segment (as it would be incorrect).
#
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE x6 USING fts4;
INSERT INTO x6 SELECT words FROM t1;
INSERT INTO x6 SELECT words FROM t1;
INSERT INTO x6 SELECT words FROM t1;
INSERT INTO x6 SELECT words FROM t1;
INSERT INTO x6 SELECT words FROM t1;
INSERT INTO x6 SELECT words FROM t1;
SELECT level, idx, end_block FROM x6_segdir;
} {
0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006}
0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
}
do_execsql_test 7.2 {
INSERT INTO x6(x6) VALUES('merge=25,4');
SELECT level, idx, end_block FROM x6_segdir;
} {
0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006}
0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
1 0 {16014 -51226}
}
do_execsql_test 7.3 {
UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1;
SELECT level, idx, end_block FROM x6_segdir;
} {
0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006}
0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
1 0 16014
}
do_execsql_test 7.4 {
INSERT INTO x6(x6) VALUES('merge=25,4');
SELECT level, idx, end_block FROM x6_segdir;
} {
0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006}
0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
1 0 16014
}
do_execsql_test 7.5 {
INSERT INTO x6(x6) VALUES('merge=2500,4');
SELECT level, idx, end_block FROM x6_segdir;
} {
0 0 {598 118006} 0 1 {718 118006} 1 0 16014
}
do_execsql_test 7.6 {
INSERT INTO x6(x6) VALUES('merge=2500,2');
SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
} {
2 0 23695 24147 {41262 633507}
}
do_execsql_test 7.7 {
SELECT sum(length(block)) FROM x6_segments
WHERE blockid BETWEEN 23695 AND 24147
} {633507}
finish_test

93
test/fts4growth2.test Normal file
View File

@ -0,0 +1,93 @@
# 2014 May 12
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS4 module.
#
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts4growth2
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
source $testdir/genesis.tcl
do_execsql_test 1.0 { CREATE TABLE t1(docid, words); }
fts_kjv_genesis
proc structure {} {
puts [ db eval {SELECT level, count(*) FROM x1_segdir GROUP BY level} ]
}
proc tt {val} {
execsql {
DELETE FROM x1
WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0);
}
execsql {
INSERT INTO x1(docid, content)
SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0;
}
}
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE x1 USING fts4;
INSERT INTO x1(x1) VALUES('automerge=2');
}
do_test 1.2 {
for {set i 0} {$i < 40} {incr i} {
tt 0 ; tt 1 ; tt 2 ; tt 3
}
execsql {
SELECT max(level) FROM x1_segdir;
SELECT count(*) FROM x1_segdir WHERE level=2;
}
} {2 1}
do_test 1.3 {
for {set i 0} {$i < 40} {incr i} {
tt 0 ; tt 1 ; tt 2 ; tt 3
}
execsql {
SELECT max(level) FROM x1_segdir;
SELECT count(*) FROM x1_segdir WHERE level=2;
}
} {2 1}
#-------------------------------------------------------------------------
#
do_execsql_test 2.1 {
DELETE FROM t1 WHERE rowid>16;
DROP TABLE IF EXISTS x1;
CREATE VIRTUAL TABLE x1 USING fts4;
}
db func second second
proc second {L} {lindex $L 1}
for {set tn 0} {$tn < 40} {incr tn} {
do_test 2.2.$tn {
for {set i 0} {$i < 100} {incr i} {
tt 0 ; tt 1 ; tt 2 ; tt 3
}
execsql { SELECT max(level) FROM x1_segdir }
} {1}
}
finish_test

44
test/fts4merge4.test
View File

@ -53,6 +53,50 @@ do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35
do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {}
do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1
#-------------------------------------------------------------------------
# Now test that the automerge=? option appears to work.
#
do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; }
set doc ""
foreach c1 "a b c d e f g h i j" {
foreach c2 "a b c d e f g h i j" {
foreach c3 "a b c d e f g h i j" {
lappend doc "$c1$c2$c3"
}
}
}
set doc [string repeat $doc 10]
foreach {tn am expected} {
1 {automerge=2} {1 1 2 1 4 1 6 1}
2 {automerge=4} {1 2 2 1 3 1}
3 {automerge=8} {0 4 1 3 2 1}
4 {automerge=1} {0 4 1 3 2 1}
} {
foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} {
do_test 2.2.$tn.$tn2 {
execsql { DELETE FROM t2 }
execsql { INSERT INTO t2(t2) VALUES($am) };
eval $openclose
for {set i 0} {$i < 100} {incr i} {
execsql {
BEGIN;
INSERT INTO t2 VALUES($doc);
INSERT INTO t2 VALUES($doc);
INSERT INTO t2 VALUES($doc);
INSERT INTO t2 VALUES($doc);
INSERT INTO t2 VALUES($doc);
COMMIT;
}
}
execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level }
} [list {*}$expected]
}
}
sqlite3_enable_shared_cache $::enable_shared_cache
finish_test

3
test/permutations.test
View File

@ -112,7 +112,7 @@ set allquicktests [test_set $alltests -exclude {
incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
vtab_err.test walslow.test walcrash.test walcrash3.test
walthread.test rtree3.test indexfault.test securedel2.test
sort3.test sort4.test
sort3.test sort4.test fts4growth.test fts4growth2.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
@ -197,6 +197,7 @@ test_suite "fts3" -prefix "" -description {
fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
fts4check.test fts4unicode.test fts4noti.test
fts3varint.test
fts4growth.test fts4growth2.test
}
test_suite "nofaultsim" -prefix "" -description {

1
test/trace2.test
View File

@ -136,6 +136,7 @@ ifcapable fts3 {
"-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1"
"-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
"-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
"-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC"
}
do_trace_test 2.3 {