mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changes to improve performance and support LIMIT clauses on fts3 tables. This branch is unstable for now.

Browse Source 
FossilOrigin-Name: 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9
This commit is contained in:
dan 2011-06-02 19:57:24 +00:00
parent 382874fc5c
commit e414854800
11 changed files with 1274 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1120
ext/fts3/fts3.c
View File

File diff suppressed because it is too large Load Diff

86
ext/fts3/fts3Int.h
View File

@ -47,6 +47,11 @@
*/
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
#ifndef MIN
# define MIN(x,y) ((x)<(y)?(x):(y))
#endif
/*
** Maximum length of a varint encoded integer. The varint format is different
** from that used by SQLite, so the maximum length is 10, not 9.
@ -142,10 +147,11 @@ typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;
typedef struct Fts3PhraseToken Fts3PhraseToken;
typedef struct Fts3Doclist Fts3Doclist;
typedef struct Fts3SegFilter Fts3SegFilter;
typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
typedef struct Fts3MultiSegReader Fts3MultiSegReader;
/*
** A connection to a fulltext index is an instance of the following
@ -224,6 +230,7 @@ struct Fts3Cursor {
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
int bIncremental; /* True to use incremental querying */
int nPhrase; /* Number of matchable phrases in query */
Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
@ -263,6 +270,17 @@ struct Fts3Cursor {
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
struct Fts3Doclist {
char *aAll; /* Array containing doclist (or NULL) */
int nAll; /* Size of a[] in bytes */
sqlite3_int64 iDocid; /* Current docid (if p!=0) */
char *pNextDocid; /* Pointer to next docid */
char *pList; /* Pointer to position list following iDocid */
int nList; /* Length of position list */
} doclist;
/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence. A single token is the base case and the most common case.
@ -277,23 +295,29 @@ struct Fts3PhraseToken {
/* Variables above this point are populated when the expression is
** parsed (by code in fts3_expr.c). Below this point the variables are
** used when evaluating the expression. */
int bFulltext; /* True if full-text index was used */
Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
};
struct Fts3Phrase {
/* Variables populated by fts3_expr.c when parsing a MATCH expression */
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
/* Cache of doclist for this phrase. */
Fts3Doclist doclist;
int bIncr; /* True if doclist is loaded incrementally */
#if 1
int isLoaded; /* True if aDoclist/nDoclist are initialized. */
char *aDoclist; /* Buffer containing doclist */
int nDoclist; /* Size of aDoclist in bytes */
sqlite3_int64 iCurrent;
char *pCurrent;
#endif
/* Variables below this point are populated by fts3_expr.c when parsing
** a MATCH expression. Everything above is part of the evaluation phase.
*/
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
};
@ -317,6 +341,12 @@ struct Fts3Expr {
Fts3Expr *pLeft; /* Left operand */
Fts3Expr *pRight; /* Right operand */
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
/* The following are used by the fts3_eval.c module. */
sqlite3_int64 iDocid; /* Current docid */
u8 bEof; /* True this expression is at EOF already */
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
};
/*
@ -366,11 +396,12 @@ void sqlite3Fts3SegmentsClose(Fts3Table *);
#define FTS3_SEGCURSOR_PENDING -1
#define FTS3_SEGCURSOR_ALL -2
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
int sqlite3Fts3SegReaderCursor(
Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *);
Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@ -387,7 +418,7 @@ struct Fts3SegFilter {
int flags;
};
struct Fts3SegReaderCursor {
struct Fts3MultiSegReader {
/* Used internally by sqlite3Fts3SegReaderXXX() calls */
Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
int nSegment; /* Size of apSegment array */
@ -396,8 +427,11 @@ struct Fts3SegReaderCursor {
char *aBuffer; /* Buffer to merge doclists in */
int nBuffer; /* Allocated size of aBuffer[] in bytes */
/* Cost of running this iterator. Used by fts3.c only. */
int nCost;
int iColFilter; /* If >=0, filter for this column */
/* Used by fts3.c only. */
int nCost; /* Cost of running iterator */
int bLookup; /* True if a lookup of a single entry. */
/* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
char *zTerm; /* Pointer to term buffer */
@ -413,7 +447,6 @@ int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
@ -446,4 +479,29 @@ int sqlite3Fts3InitTerm(sqlite3 *db);
/* fts3_aux.c */
int sqlite3Fts3InitAux(sqlite3 *db);
int sqlite3Fts3TermSegReaderCursor(
Fts3Cursor *pCsr, /* Virtual table cursor handle */
const char *zTerm, /* Term to query for */
int nTerm, /* Size of zTerm in bytes */
int isPrefix, /* True for a prefix search */
Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
);
int sqlite3Fts3EvalPhraseCache(Fts3Cursor *, Fts3Phrase *);
sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*);
void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr);
int sqlite3Fts3MsrIncrStart(
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
char *sqlite3Fts3EvalPhrasePoslist(
Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol);
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
#endif /* _FTSINT_H */

2
ext/fts3/fts3_aux.c
View File

@ -28,7 +28,7 @@ struct Fts3auxTable {
struct Fts3auxCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */
Fts3MultiSegReader csr; /* Must be right after "base" */
Fts3SegFilter filter;
char *zStop;
int nStop; /* Byte-length of string zStop */

5
ext/fts3/fts3_expr.c
View File

@ -768,7 +768,10 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){
assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
sqlite3Fts3ExprFree(p->pLeft);
sqlite3Fts3ExprFree(p->pRight);
if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist);
if( p->pPhrase ){
sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
sqlite3_free(p->pPhrase->aDoclist);
}
sqlite3_free(p);
}
}

110
ext/fts3/fts3_snippet.c
View File

@ -416,7 +416,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
pPhrase->nToken = pExpr->pPhrase->nToken;
pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol);
if( pCsr ){
int iFirst = 0;
pPhrase->pList = pCsr;
@ -826,46 +826,31 @@ static int fts3ExprGlobalHitsCb(
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
Fts3Cursor *pCsr = p->pCursor;
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter;
char *pEnd;
char *pFree = 0;
u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
assert( pPhrase->isLoaded );
if( pCsr->pDeferred ){
int ii;
for(ii=0; ii<pPhrase->nToken; ii++){
if( pPhrase->aToken[ii].bFulltext ) break;
}
if( ii<pPhrase->nToken ){
int nFree = 0;
int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
if( rc!=SQLITE_OK ) return rc;
pIter = pFree;
pEnd = &pFree[nFree];
}else{
int iCol; /* Column index */
for(iCol=0; iCol<p->nCol; iCol++){
aOut[iCol*3 + 1] = (u32)p->nDoc;
aOut[iCol*3 + 2] = (u32)p->nDoc;
}
return SQLITE_OK;
if( pExpr->bDeferred ){
int iCol; /* Column index */
for(iCol=0; iCol<p->nCol; iCol++){
aOut[iCol*3 + 1] = (u32)p->nDoc;
aOut[iCol*3 + 2] = (u32)p->nDoc;
}
}else{
pIter = pPhrase->aDoclist;
pEnd = &pPhrase->aDoclist[pPhrase->nDoclist];
char *pIter;
char *pEnd;
int n;
int rc = sqlite3Fts3EvalPhraseDoclist(
p->pCursor, pExpr, (const char **)&pIter, &n
);
if( rc!=SQLITE_OK ) return rc;
pEnd = &pIter[n];
/* Fill in the global hit count matrix row for this phrase. */
while( pIter<pEnd ){
while( *pIter++ & 0x80 ); /* Skip past docid. */
fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
}
}
/* Fill in the global hit count matrix row for this phrase. */
while( pIter<pEnd ){
while( *pIter++ & 0x80 ); /* Skip past docid. */
fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
}
sqlite3_free(pFree);
return SQLITE_OK;
}
@ -882,15 +867,15 @@ static int fts3ExprLocalHitsCb(
MatchInfo *p = (MatchInfo *)pCtx;
int iStart = iPhrase * p->nCol * 3;
int i;
sqlite3_int64 iDocid = p->pCursor->iPrevId;
for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
if( pExpr->pPhrase->aDoclist ){
for(i=0; i<p->nCol; i++){
char *pCsr;
pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i);
if( pCsr ){
fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
}else{
p->aMatchinfo[iStart+i*3] = 0;
}
}
@ -976,9 +961,8 @@ static int fts3MatchinfoSelectDoctotal(
typedef struct LcsIterator LcsIterator;
struct LcsIterator {
Fts3Expr *pExpr; /* Pointer to phrase expression */
char *pRead; /* Cursor used to iterate through aDoclist */
int iPosOffset; /* Tokens count up to end of this phrase */
int iCol; /* Current column number */
char *pRead; /* Cursor used to iterate through aDoclist */
int iPos; /* Current position */
};
@ -1009,17 +993,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){
int rc = 0;
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
if( iRead==0 ){
pIter->iCol = LCS_ITERATOR_FINISHED;
if( iRead==0 || iRead==1 ){
pRead = 0;
rc = 1;
}else{
if( iRead==1 ){
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
pIter->iCol = (int)iRead;
pIter->iPos = pIter->iPosOffset;
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
rc = 1;
}
pIter->iPos += (int)(iRead-2);
}
@ -1043,6 +1020,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
int i;
int iCol;
int nToken = 0;
sqlite3_int64 iDocid = pCsr->iPrevId;
/* Allocate and populate the array of LcsIterator objects. The array
** contains one element for each matchable phrase in the query.
@ -1051,42 +1029,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
if( !aIter ) return SQLITE_NOMEM;
memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
(void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i];
nToken -= pIter->pExpr->pPhrase->nToken;
pIter->iPosOffset = nToken;
pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
if( pIter->pRead ){
pIter->iPos = pIter->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
}else{
pIter->iCol = LCS_ITERATOR_FINISHED;
}
}
for(iCol=0; iCol<pInfo->nCol; iCol++){
int nLcs = 0; /* LCS value for this column */
int nLive = 0; /* Number of iterators in aIter not at EOF */
/* Loop through the iterators in aIter[]. Set nLive to the number of
** iterators that point to a position-list corresponding to column iCol.
*/
for(i=0; i<pInfo->nPhrase; i++){
assert( aIter[i].iCol>=iCol );
if( aIter[i].iCol==iCol ) nLive++;
LcsIterator *pIt = &aIter[i];
pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol);
if( pIt->pRead ){
pIt->iPos = pIt->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
nLive++;
}
}
/* The following loop runs until all iterators in aIter[] have finished
** iterating through positions in column iCol. Exactly one of the
** iterators is advanced each time the body of the loop is run.
*/
while( nLive>0 ){
LcsIterator *pAdv = 0; /* The iterator to advance by one position */
int nThisLcs = 0; /* LCS for the current iterator positions */
for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i];
if( iCol!=pIter->iCol ){
if( pIter->pRead==0 ){
/* This iterator is already at EOF for this column. */
nThisLcs = 0;
}else{
@ -1426,7 +1396,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
int iPos = 0; /* First position in position-list */
UNUSED_PARAMETER(iPhrase);
pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol);
nTerm = pExpr->pPhrase->nToken;
if( pList ){
fts3GetDeltaPosition(&pList, &iPos);

2
ext/fts3/fts3_term.c
View File

@ -33,7 +33,7 @@ struct Fts3termTable {
struct Fts3termCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */
Fts3MultiSegReader csr; /* Must be right after "base" */
Fts3SegFilter filter;
int isEof; /* True if cursor is at EOF */

170
ext/fts3/fts3_write.c
View File

@ -1155,6 +1155,8 @@ int sqlite3Fts3SegReaderCost(
int nCost = 0; /* Cost in bytes to return */
int pgsz = p->nPgsz; /* Database page size */
assert( pgsz>0 );
/* If this seg-reader is reading the pending-terms table, or if all data
** for the segment is stored on the root page of the b-tree, then the cost
** is zero. In this case all required data is already in main memory.
@ -1223,6 +1225,40 @@ int sqlite3Fts3SegReaderCost(
return rc;
}
int sqlite3Fts3MsrOvfl(
Fts3Cursor *pCsr,
Fts3MultiSegReader *pMsr,
int *pnOvfl
){
Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
int nOvfl = 0;
int ii;
int rc = SQLITE_OK;
int pgsz = p->nPgsz;
assert( p->bHasStat );
assert( pgsz>0 );
for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
Fts3SegReader *pReader = pMsr->apSegment[ii];
if( !fts3SegReaderIsPending(pReader)
&& !fts3SegReaderIsRootOnly(pReader)
){
int jj;
for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
int nBlob;
rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob);
if( rc!=SQLITE_OK ) break;
if( (nBlob+35)>pgsz ){
nOvfl += (nBlob + 34)/pgsz;
}
}
}
}
*pnOvfl = nOvfl;
return rc;
}
/*
** Free all allocations associated with the iterator passed as the
** second argument.
@ -2140,9 +2176,107 @@ static void fts3ColumnFilter(
*pnList = nList;
}
int sqlite3Fts3MsrIncrStart(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr, /* Cursor object */
int iCol, /* Column to match on. */
const char *zTerm, /* Term to iterate through a doclist for */
int nTerm /* Number of bytes in zTerm */
){
int i;
int nSegment = pCsr->nSegment;
assert( pCsr->pFilter==0 );
assert( zTerm && nTerm>0 );
/* Advance each segment iterator until it points to the term zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
do {
int rc = fts3SegReaderNext(p, pSeg);
if( rc!=SQLITE_OK ) return rc;
}while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
}
fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
/* Determine how many of the segments actually point to zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
break;
}
}
pCsr->nAdvance = i;
/* Advance each of the segments to point to the first docid. */
for(i=0; i<pCsr->nAdvance; i++){
fts3SegReaderFirstDocid(pCsr->apSegment[i]);
}
assert( iCol<0 || iCol<p->nColumn );
pCsr->iColFilter = iCol;
return SQLITE_OK;
}
int sqlite3Fts3MsrIncrNext(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
sqlite3_int64 *piDocid, /* OUT: Docid value */
char **paPoslist, /* OUT: Pointer to position list */
int *pnPoslist /* OUT: Size of position list in bytes */
){
int rc = SQLITE_OK;
int nMerge = pMsr->nAdvance;
Fts3SegReader **apSegment = pMsr->apSegment;
if( nMerge==0 ){
*paPoslist = 0;
return SQLITE_OK;
}
while( 1 ){
Fts3SegReader *pSeg;
fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
pSeg = pMsr->apSegment[0];
if( pSeg->pOffsetList==0 ){
*paPoslist = 0;
break;
}else{
char *pList;
int nList;
int j;
sqlite3_int64 iDocid = apSegment[0]->iDocid;
fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
j = 1;
while( j<nMerge
&& apSegment[j]->pOffsetList
&& apSegment[j]->iDocid==iDocid
){
fts3SegReaderNextDocid(apSegment[j], 0, 0);
}
if( pMsr->iColFilter>=0 ){
fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
}
if( nList>0 ){
*piDocid = iDocid;
*paPoslist = pList;
*pnPoslist = nList;
break;
}
}
}
return rc;
}
int sqlite3Fts3SegReaderStart(
Fts3Table *p, /* Virtual table handle */
Fts3SegReaderCursor *pCsr, /* Cursor object */
Fts3MultiSegReader *pCsr, /* Cursor object */
Fts3SegFilter *pFilter /* Restrictions on range of iteration */
){
int i;
@ -2173,7 +2307,7 @@ int sqlite3Fts3SegReaderStart(
int sqlite3Fts3SegReaderStep(
Fts3Table *p, /* Virtual table handle */
Fts3SegReaderCursor *pCsr /* Cursor object */
Fts3MultiSegReader *pCsr /* Cursor object */
){
int rc = SQLITE_OK;
@ -2308,8 +2442,9 @@ int sqlite3Fts3SegReaderStep(
return rc;
}
void sqlite3Fts3SegReaderFinish(
Fts3SegReaderCursor *pCsr /* Cursor object */
Fts3MultiSegReader *pCsr /* Cursor object */
){
if( pCsr ){
int i;
@ -2342,7 +2477,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
int iNewLevel = 0; /* Level/index to create new segment at */
SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
Fts3SegFilter filter; /* Segment term filter condition */
Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */
Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
int bIgnoreEmpty = 0; /* True to ignore empty segments */
assert( iLevel==FTS3_SEGCURSOR_ALL
@ -2746,6 +2881,33 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
return rc;
}
int sqlite3Fts3DeferredTokenList(
Fts3DeferredToken *p,
char **ppData,
int *pnData
){
char *pRet;
int nSkip;
sqlite3_int64 dummy;
*ppData = 0;
*pnData = 0;
if( p->pList==0 ){
return SQLITE_OK;
}
pRet = (char *)sqlite3_malloc(p->pList->nData);
if( !pRet ) return SQLITE_NOMEM;
nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
*pnData = p->pList->nData - nSkip;
*ppData = pRet;
memcpy(pRet, &p->pList->aData[nSkip], *pnData);
return SQLITE_OK;
}
/*
** Add an entry for token pToken to the pCsr->pDeferred list.
*/

28
manifest
View File

@ -1,5 +1,5 @@
C Minor\schanges\smade\swhile\splanning\sa\slarger\schange.
D 2011-05-28T15:57:40.694
C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow.
D 2011-06-02T19:57:24.733
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c eb59cdd89e9309ab9b2dca196a7c52f9e8927319
F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 832f4d421f03a9d364186e779ee51994df500c62
F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b
F ext/fts3/fts3_expr.c 25e30cf24198333f2ed545af905b168e88f56903
F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50
F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f
F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2
F ext/fts3/fts3_snippet.c 6ee626017ddcf7d72ca4f587724e3506434fc0d7
F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501
F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274
F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1
F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c ed58c53fbcbc2ea79258e734159a5951ffeb1bd4
F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -463,7 +463,7 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32
F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f
F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35
F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
@ -612,7 +612,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6
F test/permutations.test d27eac16dae111ff7cec331dab4bca08625ba65a
F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850
F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P cc83991caae7c7d647432d5711b6cd80228c3002
R eef52aec1faead4921ebaf39c1605d2b
P 84097a4c759b1d65890af885f137d3cb16eef584
R e38e0cc84edbfdb5eae395b3be2f7a86
U dan
Z 01d8f7aea2ce8ebb35c05ccc519b614f
Z 53d59cf3dcd8991c66b0afd5fb898b1a

2
manifest.uuid
View File

@ -1 +1 @@
84097a4c759b1d65890af885f137d3cb16eef584
28149a7882a1e9dfe4a75ec5b91d176ebe6284e9

29
test/fts3defer.test
View File

@ -20,6 +20,8 @@ ifcapable !fts3 {
set sqlite_fts3_enable_parentheses 1
set fts3_simple_deferred_tokens_only 1
set ::testprefix fts3defer
#--------------------------------------------------------------------------
@ -257,7 +259,6 @@ foreach {tn setup} {
do_select_test 1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
} {100}
if {$tn==3} breakpoint
do_select_test 1.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
} {7 70 98}
@ -282,13 +283,16 @@ if {$tn==3} breakpoint
do_select_test 1.10 {
SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
do_select_test 1.11 {
SELECT rowid FROM t1
WHERE t1 MATCH '(
zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR
zk OR zkhdvkw OR zm OR zsmhnf
) vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
if { $fts3_simple_deferred_tokens_only==0 } {
do_select_test 1.11 {
SELECT rowid FROM t1
WHERE t1 MATCH '(
zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR
zk OR zkhdvkw OR zm OR zsmhnf
) vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
}
do_select_test 2.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
@ -364,6 +368,7 @@ if {$tn==3} breakpoint
foreach DO_MALLOC_TEST $dmt_modes {
# Phrase search.
#
do_select_test 5.$DO_MALLOC_TEST.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"'
} {8 15 36 64 67 72}
@ -416,9 +421,11 @@ if {$tn==3} breakpoint
do_select_test 6.2.2 {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"'
} {15 26 92 96}
do_select_test 6.2.3 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
if {$fts3_simple_deferred_tokens_only==0} {
do_select_test 6.2.3 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
}
}
set testprefix fts3defer

1
test/permutations.test
View File

@ -179,6 +179,7 @@ test_suite "fts3" -prefix "" -description {
fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test
fts3near.test fts3query.test fts3shared.test fts3snippet.test
fts3sort.test
fts3fault.test fts3malloc.test fts3matchinfo.test