diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index ff217c881b..5f55bbadb5 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -67,6 +67,39 @@ void sqlite3Fts5Dequote(char *z); ** End of interface to code in fts5_config.c. **************************************************************************/ +/************************************************************************** +*/ + +/* +** Buffer object for the incremental building of string data. +*/ +typedef struct Fts5Buffer Fts5Buffer; +struct Fts5Buffer { + u8 *p; + int n; + int nSpace; +}; + +int sqlite3Fts5BufferGrow(int*, Fts5Buffer*, int); +void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); +void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); +void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); +void sqlite3Fts5BufferFree(Fts5Buffer*); +void sqlite3Fts5BufferZero(Fts5Buffer*); +void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); +void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); + +#define fts5BufferZero(x) sqlite3Fts5BufferZero(x) +#define fts5BufferGrow(a,b,c) sqlite3Fts5BufferGrow(a,b,c) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferFree(a) sqlite3Fts5BufferFree(a) +#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) +#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) + +/* +** End of interface to code in fts5_buffer.c. +**************************************************************************/ + /************************************************************************** ** Interface to code in fts5_index.c. fts5_index.c contains contains code ** to access the data stored in the %_data table. diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c new file mode 100644 index 0000000000..d8ad29f59a --- /dev/null +++ b/ext/fts5/fts5_buffer.c @@ -0,0 +1,139 @@ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + + + +#include "fts5Int.h" + +int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){ + /* A no-op if an error has already occurred */ + if( *pRc ) return 1; + + if( (pBuf->n + nByte) > pBuf->nSpace ){ + u8 *pNew; + int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; + while( nNew<(pBuf->n + nByte) ){ + nNew = nNew * 2; + } + pNew = sqlite3_realloc(pBuf->p, nNew); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return 1; + }else{ + pBuf->nSpace = nNew; + pBuf->p = pNew; + } + } + return 0; +} + +/* +** Encode value iVal as an SQLite varint and append it to the buffer object +** pBuf. If an OOM error occurs, set the error code in p. +*/ +void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ + if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return; + pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal); +} + +/* +** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +void sqlite3Fts5BufferAppendBlob( + int *pRc, + Fts5Buffer *pBuf, + int nData, + const u8 *pData +){ + if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return; + memcpy(&pBuf->p[pBuf->n], pData, nData); + pBuf->n += nData; +} + +/* +** Append the nul-terminated string zStr to the buffer pBuf. This function +** ensures that the byte following the buffer data is set to 0x00, even +** though this byte is not included in the pBuf->n count. +*/ +void sqlite3Fts5BufferAppendString( + int *pRc, + Fts5Buffer *pBuf, + const char *zStr +){ + int nStr = strlen(zStr); + if( sqlite3Fts5BufferGrow(pRc, pBuf, nStr+1) ) return; + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr, (const u8*)zStr); + if( *pRc==SQLITE_OK ) pBuf->p[pBuf->n] = 0x00; +} + +/* +** Argument zFmt is a printf() style format string. This function performs +** the printf() style processing, then appends the results to buffer pBuf. +** +** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte +** following the buffer data is set to 0x00, even though this byte is not +** included in the pBuf->n count. +*/ +void sqlite3Fts5BufferAppendPrintf( + int *pRc, + Fts5Buffer *pBuf, + char *zFmt, ... +){ + if( *pRc==SQLITE_OK ){ + char *zTmp; + va_list ap; + va_start(ap, zFmt); + zTmp = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + + if( zTmp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp); + sqlite3_free(zTmp); + } + } +} + +/* +** Free any buffer allocated by pBuf. Zero the structure before returning. +*/ +void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ + sqlite3_free(pBuf->p); + memset(pBuf, 0, sizeof(Fts5Buffer)); +} + +/* +** Zero the contents of the buffer object. But do not free the associated +** memory allocation. +*/ +void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ + pBuf->n = 0; +} + +/* +** Set the buffer to contain nData/pData. If an OOM error occurs, leave an +** the error code in p. If an error has already occurred when this function +** is called, it is a no-op. +*/ +void sqlite3Fts5BufferSet( + int *pRc, + Fts5Buffer *pBuf, + int nData, + const u8 *pData +){ + pBuf->n = 0; + sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData); +} diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 364ac024ef..1c0e936cb0 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -68,6 +68,8 @@ struct Fts5ExprTerm { ** within a document for it to match. */ struct Fts5ExprPhrase { + Fts5Buffer poslist; /* Current position list */ + i64 iRowid; /* Current rowid */ int nTerm; /* Number of entries in aTerm[] */ Fts5ExprTerm aTerm[0]; /* Terms that make up this phrase */ }; @@ -94,6 +96,43 @@ struct Fts5Parse { Fts5ExprNode *pExpr; /* Result of a successful parse */ }; +/************************************************************************* +*/ +typedef struct Fts5PoslistIter Fts5PoslistIter; +struct Fts5PoslistIter { + const u8 *a; /* Position list to iterate through */ + int n; /* Size of buffer at a[] in bytes */ + int i; /* Current offset in a[] */ + + /* Output variables */ + int bEof; /* Set to true at EOF */ + i64 iPos; /* (iCol<<32) + iPos */ +}; + +static void fts5PoslistIterNext(Fts5PoslistIter *pIter){ + if( pIter->i>=pIter->n ){ + pIter->bEof = 1; + }else{ + int iVal; + pIter->i += getVarint32(&pIter->a[pIter->i], iVal); + if( iVal==1 ){ + pIter->i += getVarint32(&pIter->a[pIter->i], iVal); + pIter->iPos = ((u64)iVal << 32); + pIter->i += getVarint32(&pIter->a[pIter->i], iVal); + } + pIter->iPos += (iVal-2); + } +} + +static void fts5PoslistIterInit(const u8 *a, int n, Fts5PoslistIter *pIter){ + memset(pIter, 0, sizeof(*pIter)); + pIter->a = a; + pIter->n = n; + fts5PoslistIterNext(pIter); +} +/* +*************************************************************************/ + void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ if( pParse->rc==SQLITE_OK ){ va_list ap; @@ -246,6 +285,259 @@ static int fts5ExprNodeTest(Fts5Expr *pExpr, Fts5ExprNode *pNode){ assert( 0 ); return SQLITE_OK; } + +/* +** All individual term iterators in pPhrase are guaranteed to be valid and +** pointing to the same rowid when this function is called. This function +** checks if the current rowid really is a match, and if so populates +** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch +** is set to true if this is really a match, or false otherwise. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if the current rowid is +** not a match. +*/ +static int fts5ExprPhraseIsMatch( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbMatch /* OUT: Set to true if really a match */ +){ + Fts5PoslistIter aStatic[4]; + Fts5PoslistIter *aIter = aStatic; + int i; + int rc = SQLITE_OK; + + if( pPhrase->nTerm>(sizeof(aStatic) / sizeof(aStatic[0])) ){ + int nByte = sizeof(Fts5PoslistIter) * pPhrase->nTerm; + aIter = (Fts5PoslistIter*)sqlite3_malloc(nByte); + if( !aIter ) return SQLITE_NOMEM; + } + + /* Initialize a term iterator for each term in the phrase */ + for(i=0; inTerm; i++){ + int n; + const u8 *a = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &n); + fts5PoslistIterInit(a, n, &aIter[i]); + } + + *pbMatch = 0; + while( 1 ){ + + int bMatch = 1; + i64 iPos = aIter[0].iPos; + for(i=1; inTerm; i++){ + Fts5PoslistIter *pPos = &aIter[i]; + i64 iAdj = pPos->iPos-i; + if( (pPos->iPos-i)!=iPos ){ + bMatch = 0; + if( iAdj>iPos ) iPos = iAdj; + } + } + if( bMatch ){ + *pbMatch = 1; + break; + } + + for(i=0; inTerm; i++){ + Fts5PoslistIter *pPos = &aIter[i]; + while( (pPos->iPos-i) < iPos ){ + fts5PoslistIterNext(pPos); + if( pPos->bEof ) goto ismatch_out; + } + } + } + + ismatch_out: + if( aIter!=aStatic ) sqlite3_free(aIter); + return rc; +} + +/* +** All individual term iterators in pPhrase are guaranteed to be valid when +** this function is called. This function checks if all term iterators +** point to the same rowid, and if not, advances them until they do. +** If an EOF is reached before this happens, *pbEof is set to true before +** returning. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if an iterator reaches +** EOF. +*/ +static int fts5ExprPhraseNextRowidMatch( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbEof /* OUT: Set to true if phrase at EOF */ +){ + assert( *pbEof==0 ); + while( 1 ){ + int i; + int bMatch = 1; + i64 iMin = sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter); + for(i=1; inTerm; i++){ + i64 iRowid = sqlite3Fts5IterRowid(pPhrase->aTerm[i].pIter); + if( iRowid!=iMin ){ + bMatch = 0; + if( iRowidnTerm; i++){ + Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter; + while( sqlite3Fts5IterRowid(pIter)>iMin ){ + sqlite3Fts5IterNext(pIter, 0); + if( sqlite3Fts5IterEof(pIter) ){ + *pbEof = 1; + return SQLITE_OK; + } + } + } + } + + return SQLITE_OK; +} + +static int fts5ExprPhraseAdvanceAll( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbEof /* OUT: Set to true if phrase at EOF */ +){ + int i; + int rc = SQLITE_OK; + for(i=0; inTerm; i++){ + Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter; + sqlite3Fts5IterNext(pIter, 0); + if( sqlite3Fts5IterEof(pIter) ){ + *pbEof = 1; + break; + } + } + return rc; +} + +/* +** Argument pPhrase points to a multi-term phrase object. All individual +** term iterators point to valid entries (not EOF). +* +** This function tests if the term iterators currently all point to the +** same rowid, and if so, if the rowid matches the phrase constraint. If +** so, the pPhrase->poslist buffer is populated and the pPhrase->iRowid +** variable set before returning. Or, if the current combination of +** iterators is not a match, they are advanced until they are. If one of +** the iterators reaches EOF before a match is found, *pbEof is set to +** true before returning. The final values of the pPhrase->poslist and +** iRowid fields are undefined in this case. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if an iterator reaches +** EOF. +*/ +static int fts5ExprPhraseNextMatch( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbEof /* OUT: Set to true if phrase at EOF */ +){ + int i; /* Used to iterate through terms */ + int rc = SQLITE_OK; /* Return code */ + int bMatch = 0; + + assert( *pbEof==0 ); + + while( 1 ){ + rc = fts5ExprPhraseNextRowidMatch(pExpr, pPhrase, pbEof); + if( rc!=SQLITE_OK || *pbEof ) break; + + /* At this point, all term iterators are valid and point to the same rowid. + ** The following assert() statements verify this. */ +#ifdef SQLITE_DEBUG + for(i=0; inTerm; i++){ + Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter; + Fts5IndexIter *pOne = pPhrase->aTerm[0].pIter; + assert( 0==sqlite3Fts5IterEof(pIter) ); + assert( sqlite3Fts5IterRowid(pOne)==sqlite3Fts5IterRowid(pIter) ); + } +#endif + + rc = fts5ExprPhraseIsMatch(pExpr, pPhrase, &bMatch); + if( rc!=SQLITE_OK || bMatch ) break; + rc = fts5ExprPhraseAdvanceAll(pExpr, pPhrase, pbEof); + if( rc!=SQLITE_OK || *pbEof ) break; + } + + pPhrase->iRowid = sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter); + return rc; +} + +/* +** Advance the phrase iterator pPhrase to the next match. +*/ +static int fts5ExprPhraseNext( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbEof /* OUT: Set to true if phrase at EOF */ +){ + int i; + for(i=0; inTerm; i++){ + Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter; + sqlite3Fts5IterNext(pIter, 0); + if( sqlite3Fts5IterEof(pIter) ){ + *pbEof = 1; + return SQLITE_OK; + } + } + + if( pPhrase->nTerm==1 ){ + pPhrase->iRowid = sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter); + }else{ + fts5ExprPhraseNextMatch(pExpr, pPhrase, pbEof); + } + + return SQLITE_OK; +} + +/* +** Point phrase object pPhrase at the first matching document. Or, if there +** are no matching documents at all, move pPhrase to EOF and set *pbEof to +** true before returning. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error +** code. +*/ +static int fts5ExprPhraseFirst( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ + int *pbEof /* OUT: Set to true if phrase at EOF */ +){ + int i; /* Used to iterate through terms */ + int rc = SQLITE_OK; + + for(i=0; inTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; + pTerm->pIter = sqlite3Fts5IndexQuery( + pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0) + ); + if( sqlite3Fts5IterEof(pTerm->pIter) ){ + *pbEof = 1; + return SQLITE_OK; + } + } + + if( pPhrase->nTerm==1 ){ + const u8 *a; int n; + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + pPhrase->iRowid = sqlite3Fts5IterRowid(pIter); + a = sqlite3Fts5IterPoslist(pIter, &n); + if( a ){ + sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, n, a); + } + }else{ + rc = fts5ExprPhraseNextMatch(pExpr, pPhrase, pbEof); + } + + return rc; +} static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ int rc = SQLITE_OK; @@ -253,20 +545,10 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ pNode->bEof = 0; if( pNode->eType==FTS5_STRING ){ Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; - Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; - assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); - - pTerm->pIter = sqlite3Fts5IndexQuery( - pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm), - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | - (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0) - ); - if( sqlite3Fts5IterEof(pTerm->pIter) ){ - pNode->bEof = 1; - }else{ - pNode->iRowid = sqlite3Fts5IterRowid(pTerm->pIter); - } - + assert( pNode->pNear->nPhrase==1 ); + assert( pNode->bEof==0 ); + rc = fts5ExprPhraseFirst(pExpr, pPhrase, &pNode->bEof); + pNode->iRowid = pPhrase->iRowid; }else{ rc = fts5ExprNodeFirst(pExpr, pNode->pLeft); if( rc==SQLITE_OK ){ @@ -284,14 +566,9 @@ static int fts5ExprNodeNext(Fts5Expr *pExpr, Fts5ExprNode *pNode){ if( pNode->eType==FTS5_STRING ){ Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; - Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; - assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); - sqlite3Fts5IterNext(pTerm->pIter, 0); - if( sqlite3Fts5IterEof(pTerm->pIter) ){ - pNode->bEof = 1; - }else{ - pNode->iRowid = sqlite3Fts5IterRowid(pTerm->pIter); - } + assert( pNode->pNear->nPhrase==1 ); + rc = fts5ExprPhraseNext(pExpr, pPhrase, &pNode->bEof); + pNode->iRowid = pPhrase->iRowid; }else{ assert( 0 ); } @@ -373,6 +650,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ sqlite3Fts5IterClose(pTerm->pIter); } } + fts5BufferFree(&pPhrase->poslist); sqlite3_free(pPhrase); } } @@ -455,13 +733,13 @@ static int fts5ParseTokenize( sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew ); if( pNew==0 ) return SQLITE_NOMEM; + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); pCtx->pPhrase = pPhrase = pNew; pNew->nTerm = nNew - SZALLOC; } pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; - pTerm->bPrefix = 0; - pTerm->pIter = 0; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); pTerm->zTerm = fts5Strdup(pToken, nToken); return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 8f3ce6dca2..e60ec5f503 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -254,7 +254,6 @@ static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } typedef struct Fts5BtreeIter Fts5BtreeIter; typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; -typedef struct Fts5Buffer Fts5Buffer; typedef struct Fts5ChunkIter Fts5ChunkIter; typedef struct Fts5Data Fts5Data; typedef struct Fts5MultiSegIter Fts5MultiSegIter; @@ -298,15 +297,6 @@ struct Fts5Index { sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ }; -/* -** Buffer object for the incremental building of string data. -*/ -struct Fts5Buffer { - u8 *p; - int n; - int nSpace; -}; - struct Fts5IndexIter { Fts5Index *pIndex; Fts5Structure *pStruct; @@ -559,129 +549,6 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){ } -static int fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){ - /* A no-op if an error has already occurred */ - if( *pRc ) return 1; - - if( (pBuf->n + nByte) > pBuf->nSpace ){ - u8 *pNew; - int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; - while( nNew<(pBuf->n + nByte) ){ - nNew = nNew * 2; - } - pNew = sqlite3_realloc(pBuf->p, nNew); - if( pNew==0 ){ - *pRc = SQLITE_NOMEM; - return 1; - }else{ - pBuf->nSpace = nNew; - pBuf->p = pNew; - } - } - return 0; -} - -/* -** Encode value iVal as an SQLite varint and append it to the buffer object -** pBuf. If an OOM error occurs, set the error code in p. -*/ -static void fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ - if( fts5BufferGrow(pRc, pBuf, 9) ) return; - pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal); -} - -/* -** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set -** the error code in p. If an error has already occurred when this function -** is called, it is a no-op. -*/ -static void fts5BufferAppendBlob( - int *pRc, - Fts5Buffer *pBuf, - int nData, - const u8 *pData -){ - if( fts5BufferGrow(pRc, pBuf, nData) ) return; - memcpy(&pBuf->p[pBuf->n], pData, nData); - pBuf->n += nData; -} - -/* -** Append the nul-terminated string zStr to the buffer pBuf. This function -** ensures that the byte following the buffer data is set to 0x00, even -** though this byte is not included in the pBuf->n count. -*/ -static void fts5BufferAppendString( - int *pRc, - Fts5Buffer *pBuf, - const char *zStr -){ - int nStr = strlen(zStr); - if( fts5BufferGrow(pRc, pBuf, nStr+1) ) return; - fts5BufferAppendBlob(pRc, pBuf, nStr, (const u8*)zStr); - if( *pRc==SQLITE_OK ) pBuf->p[pBuf->n] = 0x00; -} - -/* -** Argument zFmt is a printf() style format string. This function performs -** the printf() style processing, then appends the results to buffer pBuf. -** -** Like fts5BufferAppendString(), this function ensures that the byte -** following the buffer data is set to 0x00, even though this byte is not -** included in the pBuf->n count. -*/ -static void fts5BufferAppendPrintf( - int *pRc, - Fts5Buffer *pBuf, - char *zFmt, ... -){ - if( *pRc==SQLITE_OK ){ - char *zTmp; - va_list ap; - va_start(ap, zFmt); - zTmp = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - - if( zTmp==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - fts5BufferAppendString(pRc, pBuf, zTmp); - sqlite3_free(zTmp); - } - } -} - -/* -** Free any buffer allocated by pBuf. Zero the structure before returning. -*/ -static void fts5BufferFree(Fts5Buffer *pBuf){ - sqlite3_free(pBuf->p); - memset(pBuf, 0, sizeof(Fts5Buffer)); -} - -/* -** Zero the contents of the buffer object. But do not free the associated -** memory allocation. -*/ -static void fts5BufferZero(Fts5Buffer *pBuf){ - pBuf->n = 0; -} - -/* -** Set the buffer to contain nData/pData. If an OOM error occurs, leave an -** the error code in p. If an error has already occurred when this function -** is called, it is a no-op. -*/ -static void fts5BufferSet( - int *pRc, - Fts5Buffer *pBuf, - int nData, - const u8 *pData -){ - pBuf->n = 0; - fts5BufferAppendBlob(pRc, pBuf, nData, pData); -} - /* ** Compare the contents of the pLeft buffer with the pRight/nRight blob. ** @@ -1621,6 +1488,11 @@ static void fts5ChunkIterInit( } } +static void fts5ChunkIterRelease(Fts5ChunkIter *pIter){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; +} + /* ** Read and return the next 32-bit varint from the position-list iterator ** passed as the second argument. @@ -2935,15 +2807,17 @@ static void fts5DecodeStructure( for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; - fts5BufferAppendPrintf(pRc, pBuf, " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge + ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - fts5BufferAppendPrintf(pRc, pBuf, + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, pSeg->pgnoFirst, pSeg->pgnoLast ); } - fts5BufferAppendPrintf(pRc, pBuf, "}"); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } fts5StructureRelease(p); @@ -2984,7 +2858,7 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ while( iOffpStruct = fts5StructureRead(p, 0); if( pRet->pStruct ){ fts5MultiIterNew(p, @@ -3216,8 +3095,23 @@ i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ ** disk. */ const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, int *pn){ - assert( sqlite3Fts5IterEof(pIter)==0 ); + Fts5ChunkIter iter; + Fts5Index *p = pIter->pIndex; + Fts5SegIter *pSeg = &pIter->pMulti->aSeg[ pIter->pMulti->aFirst[1] ]; + assert( sqlite3Fts5IterEof(pIter)==0 ); + fts5ChunkIterInit(p, pSeg, &iter); + if( fts5ChunkIterEof(p, &iter)==0 ){ + fts5BufferZero(&pIter->poslist); + fts5BufferGrow(&p->rc, &pIter->poslist, iter.nRem); + while( fts5ChunkIterEof(p, &iter)==0 ){ + fts5BufferAppendBlob(&p->rc, &pIter->poslist, iter.n, iter.p); + fts5ChunkIterNext(p, &iter); + } + } + fts5ChunkIterRelease(&iter); + + if( p->rc ) return 0; *pn = pIter->poslist.n; return pIter->poslist.p; } @@ -3230,6 +3124,7 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ fts5MultiIterFree(pIter->pIndex, pIter->pMulti); fts5StructureRelease(pIter->pStruct); fts5CloseReader(pIter->pIndex); + fts5BufferFree(&pIter->poslist); sqlite3_free(pIter); } } diff --git a/main.mk b/main.mk index cde67dbb2e..953d63e39f 100644 --- a/main.mk +++ b/main.mk @@ -73,6 +73,7 @@ LIBOBJ+= vdbe.o parse.o \ vdbetrace.o wal.o walker.o where.o utf.o vtab.o LIBOBJ += fts5.o +LIBOBJ += fts5_buffer.o LIBOBJ += fts5_config.o LIBOBJ += fts5_expr.o LIBOBJ += fts5_index.o @@ -572,6 +573,9 @@ rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) # FTS5 things # +fts5_buffer.o: $(TOP)/ext/fts5/fts5_buffer.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_buffer.c + fts5_config.o: $(TOP)/ext/fts5/fts5_config.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_config.c diff --git a/manifest b/manifest index e5d38010a4..2c8ddafbb9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sposition\slist\sformat\sso\sthat\sits\ssize\sin\sbytes\sis\sstored\sat\sthe\sstart\sof\sthe\slist\sitself. -D 2014-07-01T20:45:18.496 +C Add\ssupport\sfor\sphrase\squeries\sto\sfts5. +D 2014-07-02T20:18:49.027 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,10 +104,11 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 F ext/fts5/fts5.c 1af3184dd9c0e5c1686f71202d6b6cac8f225f05 -F ext/fts5/fts5Int.h 80f3d38a69a0c58ccc94428c8fc8adbcf7561a2d +F ext/fts5/fts5Int.h b7a684ff3508ab24437886f8bc873a16f494a7db +F ext/fts5/fts5_buffer.c f1a26a79e2943fe4388e531fa141941b5eb6d31a F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef -F ext/fts5/fts5_expr.c 1874b17f10a38d0b21e0c38a28637f74e4d2570a -F ext/fts5/fts5_index.c ea3dfe56a16813fcf59e03f6156965894b4b5e6f +F ext/fts5/fts5_expr.c aacfcf6b7c14ca5987ba1de0bd080eee31fca98c +F ext/fts5/fts5_index.c 6bb95f6a1ed0e50bc9f2dce7b7a92859f5821364 F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb @@ -152,7 +153,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 2bb1ec703ac4f27743961764b59cfb5f91d72bfe +F main.mk c5524f888196af43a9b5dfae878205044f549dbf F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -592,7 +593,7 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343 -F test/fts5ab.test 6436ad345d1e7eb5ab198c0174834380805f609c +F test/fts5ab.test bdc1dd9d58163c0c7b184be817f82e3bf8a81c37 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -1188,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 94eeb077d08a1d2607f3ff3a9fbf18229ba475bb -R 5d9b8f6933c58725a24e426a963b0d97 +P 62f2ff20418702ed0fbf708369edf5638445b51b +R 773d748328905c117f50682aca9f537a U dan -Z bb8816e0d501865bff7c4c8da87350cb +Z 194302280b1431e713a39a0adc2c19fe diff --git a/manifest.uuid b/manifest.uuid index ac69385214..9cec4319b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62f2ff20418702ed0fbf708369edf5638445b51b \ No newline at end of file +2e5652e6526b8fb3f5c163168d95bc0bb4c93686 \ No newline at end of file diff --git a/test/fts5ab.test b/test/fts5ab.test index 1b976caff7..1cbab9d616 100644 --- a/test/fts5ab.test +++ b/test/fts5ab.test @@ -94,11 +94,42 @@ foreach {tn expr res} { do_execsql_test 2.7.$tn { SELECT rowid FROM t1 WHERE t1 MATCH $expr } $res } -#db eval { -# SELECT fts5_decode(rowid, block) AS t FROM t1_data; -#} { -# puts $t -#} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a,b); + INSERT INTO t1(t1) VALUES('pgsz=32'); +} + +foreach {tn a b} { + 1 {abashed abandons abase abash abaft} {abases abased} + 2 {abasing abases abaft abated abandons} {abases abandoned} + 3 {abatement abash abash abated abase} {abasements abashing} + 4 {abaft abasements abase abasement abasing} {abasement abases} + 5 {abaft abashing abatement abash abasements} {abandons abandoning} + 6 {aback abate abasements abashes abandoned} {abasement abased} + 7 {abandons abated abased aback abandoning} {abases abandoned} + 8 {abashing abases abasement abaft abashing} {abashed abate} + 9 {abash abase abate abashing abashed} {abandon abandoned} + 10 {abate abandoning abandons abasement aback} {abandon abandoning} +} { + do_execsql_test 2.1.$tn.1 { INSERT INTO t1 VALUES($a, $b) } + do_execsql_test 2.1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') } +} + +foreach {tn expr res} { + 1 {abash} {9 5 3 1} + 2 {abase} {9 4 3 1} + 3 {abase + abash} {1} + 4 {abash + abase} {9} + 5 {abaft + abashing} {8 5} + 6 {abandon + abandoning} {10} + 7 {"abashing abases abasement abaft abashing"} {8} +} { + do_execsql_test 2.2.$tn { + SELECT rowid FROM t1 WHERE t1 MATCH $expr + } $res +} finish_test -