mirrors
/
sqlite
mirror of https://github.com/sqlite/sqlite
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for phrase queries to fts5. 

FossilOrigin-Name: 2e5652e6526b8fb3f5c163168d95bc0bb4c93686
This commit is contained in:
dan 2014-07-02 20:18:49 +00:00
parent acf6642819
commit 1dfacb4675
8 changed files with 569 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
ext/fts5/fts5Int.h
View File

@ -67,6 +67,39 @@ void sqlite3Fts5Dequote(char *z);
** End of interface to code in fts5_config.c.
**************************************************************************/
/**************************************************************************
*/
/*
** Buffer object for the incremental building of string data.
*/
typedef struct Fts5Buffer Fts5Buffer;
struct Fts5Buffer {
u8 *p;
int n;
int nSpace;
};
int sqlite3Fts5BufferGrow(int*, Fts5Buffer*, int);
void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
void sqlite3Fts5BufferFree(Fts5Buffer*);
void sqlite3Fts5BufferZero(Fts5Buffer*);
void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
#define fts5BufferGrow(a,b,c) sqlite3Fts5BufferGrow(a,b,c)
#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
/*
** End of interface to code in fts5_buffer.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_index.c. fts5_index.c contains contains code
** to access the data stored in the %_data table.

139
ext/fts5/fts5_buffer.c Normal file
View File

@ -0,0 +1,139 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
#include "fts5Int.h"
int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
/* A no-op if an error has already occurred */
if( *pRc ) return 1;
if( (pBuf->n + nByte) > pBuf->nSpace ){
u8 *pNew;
int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
while( nNew<(pBuf->n + nByte) ){
nNew = nNew * 2;
}
pNew = sqlite3_realloc(pBuf->p, nNew);
if( pNew==0 ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
pBuf->nSpace = nNew;
pBuf->p = pNew;
}
}
return 0;
}
/*
** Encode value iVal as an SQLite varint and append it to the buffer object
** pBuf. If an OOM error occurs, set the error code in p.
*/
void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return;
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal);
}
/*
** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
void sqlite3Fts5BufferAppendBlob(
int *pRc,
Fts5Buffer *pBuf,
int nData,
const u8 *pData
){
if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return;
memcpy(&pBuf->p[pBuf->n], pData, nData);
pBuf->n += nData;
}
/*
** Append the nul-terminated string zStr to the buffer pBuf. This function
** ensures that the byte following the buffer data is set to 0x00, even
** though this byte is not included in the pBuf->n count.
*/
void sqlite3Fts5BufferAppendString(
int *pRc,
Fts5Buffer *pBuf,
const char *zStr
){
int nStr = strlen(zStr);
if( sqlite3Fts5BufferGrow(pRc, pBuf, nStr+1) ) return;
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr, (const u8*)zStr);
if( *pRc==SQLITE_OK ) pBuf->p[pBuf->n] = 0x00;
}
/*
** Argument zFmt is a printf() style format string. This function performs
** the printf() style processing, then appends the results to buffer pBuf.
**
** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte
** following the buffer data is set to 0x00, even though this byte is not
** included in the pBuf->n count.
*/
void sqlite3Fts5BufferAppendPrintf(
int *pRc,
Fts5Buffer *pBuf,
char *zFmt, ...
){
if( *pRc==SQLITE_OK ){
char *zTmp;
va_list ap;
va_start(ap, zFmt);
zTmp = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zTmp==0 ){
*pRc = SQLITE_NOMEM;
}else{
sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
sqlite3_free(zTmp);
}
}
}
/*
** Free any buffer allocated by pBuf. Zero the structure before returning.
*/
void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
sqlite3_free(pBuf->p);
memset(pBuf, 0, sizeof(Fts5Buffer));
}
/*
** Zero the contents of the buffer object. But do not free the associated
** memory allocation.
*/
void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
pBuf->n = 0;
}
/*
** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
void sqlite3Fts5BufferSet(
int *pRc,
Fts5Buffer *pBuf,
int nData,
const u8 *pData
){
pBuf->n = 0;
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
}

326
ext/fts5/fts5_expr.c
View File

@ -68,6 +68,8 @@ struct Fts5ExprTerm {
** within a document for it to match.
*/
struct Fts5ExprPhrase {
Fts5Buffer poslist; /* Current position list */
i64 iRowid; /* Current rowid */
int nTerm; /* Number of entries in aTerm[] */
Fts5ExprTerm aTerm[0]; /* Terms that make up this phrase */
};
@ -94,6 +96,43 @@ struct Fts5Parse {
Fts5ExprNode *pExpr; /* Result of a successful parse */
};
/*************************************************************************
*/
typedef struct Fts5PoslistIter Fts5PoslistIter;
struct Fts5PoslistIter {
const u8 *a; /* Position list to iterate through */
int n; /* Size of buffer at a[] in bytes */
int i; /* Current offset in a[] */
/* Output variables */
int bEof; /* Set to true at EOF */
i64 iPos; /* (iCol<<32) + iPos */
};
static void fts5PoslistIterNext(Fts5PoslistIter *pIter){
if( pIter->i>=pIter->n ){
pIter->bEof = 1;
}else{
int iVal;
pIter->i += getVarint32(&pIter->a[pIter->i], iVal);
if( iVal==1 ){
pIter->i += getVarint32(&pIter->a[pIter->i], iVal);
pIter->iPos = ((u64)iVal << 32);
pIter->i += getVarint32(&pIter->a[pIter->i], iVal);
}
pIter->iPos += (iVal-2);
}
}
static void fts5PoslistIterInit(const u8 *a, int n, Fts5PoslistIter *pIter){
memset(pIter, 0, sizeof(*pIter));
pIter->a = a;
pIter->n = n;
fts5PoslistIterNext(pIter);
}
/*
*************************************************************************/
void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
if( pParse->rc==SQLITE_OK ){
va_list ap;
@ -246,6 +285,259 @@ static int fts5ExprNodeTest(Fts5Expr *pExpr, Fts5ExprNode *pNode){
assert( 0 );
return SQLITE_OK;
}
/*
** All individual term iterators in pPhrase are guaranteed to be valid and
** pointing to the same rowid when this function is called. This function
** checks if the current rowid really is a match, and if so populates
** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
** is set to true if this is really a match, or false otherwise.
**
** SQLITE_OK is returned if an error occurs, or an SQLite error code
** otherwise. It is not considered an error code if the current rowid is
** not a match.
*/
static int fts5ExprPhraseIsMatch(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbMatch /* OUT: Set to true if really a match */
){
Fts5PoslistIter aStatic[4];
Fts5PoslistIter *aIter = aStatic;
int i;
int rc = SQLITE_OK;
if( pPhrase->nTerm>(sizeof(aStatic) / sizeof(aStatic[0])) ){
int nByte = sizeof(Fts5PoslistIter) * pPhrase->nTerm;
aIter = (Fts5PoslistIter*)sqlite3_malloc(nByte);
if( !aIter ) return SQLITE_NOMEM;
}
/* Initialize a term iterator for each term in the phrase */
for(i=0; i<pPhrase->nTerm; i++){
int n;
const u8 *a = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &n);
fts5PoslistIterInit(a, n, &aIter[i]);
}
*pbMatch = 0;
while( 1 ){
int bMatch = 1;
i64 iPos = aIter[0].iPos;
for(i=1; i<pPhrase->nTerm; i++){
Fts5PoslistIter *pPos = &aIter[i];
i64 iAdj = pPos->iPos-i;
if( (pPos->iPos-i)!=iPos ){
bMatch = 0;
if( iAdj>iPos ) iPos = iAdj;
}
}
if( bMatch ){
*pbMatch = 1;
break;
}
for(i=0; i<pPhrase->nTerm; i++){
Fts5PoslistIter *pPos = &aIter[i];
while( (pPos->iPos-i) < iPos ){
fts5PoslistIterNext(pPos);
if( pPos->bEof ) goto ismatch_out;
}
}
}
ismatch_out:
if( aIter!=aStatic ) sqlite3_free(aIter);
return rc;
}
/*
** All individual term iterators in pPhrase are guaranteed to be valid when
** this function is called. This function checks if all term iterators
** point to the same rowid, and if not, advances them until they do.
** If an EOF is reached before this happens, *pbEof is set to true before
** returning.
**
** SQLITE_OK is returned if an error occurs, or an SQLite error code
** otherwise. It is not considered an error code if an iterator reaches
** EOF.
*/
static int fts5ExprPhraseNextRowidMatch(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbEof /* OUT: Set to true if phrase at EOF */
){
assert( *pbEof==0 );
while( 1 ){
int i;
int bMatch = 1;
i64 iMin = sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter);
for(i=1; i<pPhrase->nTerm; i++){
i64 iRowid = sqlite3Fts5IterRowid(pPhrase->aTerm[i].pIter);
if( iRowid!=iMin ){
bMatch = 0;
if( iRowid<iMin ) iMin = iRowid;
}
}
if( bMatch ) break;
for(i=0; i<pPhrase->nTerm; i++){
Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter;
while( sqlite3Fts5IterRowid(pIter)>iMin ){
sqlite3Fts5IterNext(pIter, 0);
if( sqlite3Fts5IterEof(pIter) ){
*pbEof = 1;
return SQLITE_OK;
}
}
}
}
return SQLITE_OK;
}
static int fts5ExprPhraseAdvanceAll(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbEof /* OUT: Set to true if phrase at EOF */
){
int i;
int rc = SQLITE_OK;
for(i=0; i<pPhrase->nTerm; i++){
Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter;
sqlite3Fts5IterNext(pIter, 0);
if( sqlite3Fts5IterEof(pIter) ){
*pbEof = 1;
break;
}
}
return rc;
}
/*
** Argument pPhrase points to a multi-term phrase object. All individual
** term iterators point to valid entries (not EOF).
*
** This function tests if the term iterators currently all point to the
** same rowid, and if so, if the rowid matches the phrase constraint. If
** so, the pPhrase->poslist buffer is populated and the pPhrase->iRowid
** variable set before returning. Or, if the current combination of
** iterators is not a match, they are advanced until they are. If one of
** the iterators reaches EOF before a match is found, *pbEof is set to
** true before returning. The final values of the pPhrase->poslist and
** iRowid fields are undefined in this case.
**
** SQLITE_OK is returned if an error occurs, or an SQLite error code
** otherwise. It is not considered an error code if an iterator reaches
** EOF.
*/
static int fts5ExprPhraseNextMatch(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbEof /* OUT: Set to true if phrase at EOF */
){
int i; /* Used to iterate through terms */
int rc = SQLITE_OK; /* Return code */
int bMatch = 0;
assert( *pbEof==0 );
while( 1 ){
rc = fts5ExprPhraseNextRowidMatch(pExpr, pPhrase, pbEof);
if( rc!=SQLITE_OK || *pbEof ) break;
/* At this point, all term iterators are valid and point to the same rowid.
** The following assert() statements verify this. */
#ifdef SQLITE_DEBUG
for(i=0; i<pPhrase->nTerm; i++){
Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter;
Fts5IndexIter *pOne = pPhrase->aTerm[0].pIter;
assert( 0==sqlite3Fts5IterEof(pIter) );
assert( sqlite3Fts5IterRowid(pOne)==sqlite3Fts5IterRowid(pIter) );
}
#endif
rc = fts5ExprPhraseIsMatch(pExpr, pPhrase, &bMatch);
if( rc!=SQLITE_OK || bMatch ) break;
rc = fts5ExprPhraseAdvanceAll(pExpr, pPhrase, pbEof);
if( rc!=SQLITE_OK || *pbEof ) break;
}
pPhrase->iRowid = sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter);
return rc;
}
/*
** Advance the phrase iterator pPhrase to the next match.
*/
static int fts5ExprPhraseNext(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbEof /* OUT: Set to true if phrase at EOF */
){
int i;
for(i=0; i<pPhrase->nTerm; i++){
Fts5IndexIter *pIter = pPhrase->aTerm[i].pIter;
sqlite3Fts5IterNext(pIter, 0);
if( sqlite3Fts5IterEof(pIter) ){
*pbEof = 1;
return SQLITE_OK;
}
}
if( pPhrase->nTerm==1 ){
pPhrase->iRowid = sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter);
}else{
fts5ExprPhraseNextMatch(pExpr, pPhrase, pbEof);
}
return SQLITE_OK;
}
/*
** Point phrase object pPhrase at the first matching document. Or, if there
** are no matching documents at all, move pPhrase to EOF and set *pbEof to
** true before returning.
**
** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error
** code.
*/
static int fts5ExprPhraseFirst(
Fts5Expr *pExpr, /* Expression pPhrase belongs to */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbEof /* OUT: Set to true if phrase at EOF */
){
int i; /* Used to iterate through terms */
int rc = SQLITE_OK;
for(i=0; i<pPhrase->nTerm; i++){
Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
pTerm->pIter = sqlite3Fts5IndexQuery(
pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0)
);
if( sqlite3Fts5IterEof(pTerm->pIter) ){
*pbEof = 1;
return SQLITE_OK;
}
}
if( pPhrase->nTerm==1 ){
const u8 *a; int n;
Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
pPhrase->iRowid = sqlite3Fts5IterRowid(pIter);
a = sqlite3Fts5IterPoslist(pIter, &n);
if( a ){
sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, n, a);
}
}else{
rc = fts5ExprPhraseNextMatch(pExpr, pPhrase, pbEof);
}
return rc;
}
static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
int rc = SQLITE_OK;
@ -253,20 +545,10 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
pNode->bEof = 0;
if( pNode->eType==FTS5_STRING ){
Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 );
pTerm->pIter = sqlite3Fts5IndexQuery(
pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0)
);
if( sqlite3Fts5IterEof(pTerm->pIter) ){
pNode->bEof = 1;
}else{
pNode->iRowid = sqlite3Fts5IterRowid(pTerm->pIter);
}
assert( pNode->pNear->nPhrase==1 );
assert( pNode->bEof==0 );
rc = fts5ExprPhraseFirst(pExpr, pPhrase, &pNode->bEof);
pNode->iRowid = pPhrase->iRowid;
}else{
rc = fts5ExprNodeFirst(pExpr, pNode->pLeft);
if( rc==SQLITE_OK ){
@ -284,14 +566,9 @@ static int fts5ExprNodeNext(Fts5Expr *pExpr, Fts5ExprNode *pNode){
if( pNode->eType==FTS5_STRING ){
Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 );
sqlite3Fts5IterNext(pTerm->pIter, 0);
if( sqlite3Fts5IterEof(pTerm->pIter) ){
pNode->bEof = 1;
}else{
pNode->iRowid = sqlite3Fts5IterRowid(pTerm->pIter);
}
assert( pNode->pNear->nPhrase==1 );
rc = fts5ExprPhraseNext(pExpr, pPhrase, &pNode->bEof);
pNode->iRowid = pPhrase->iRowid;
}else{
assert( 0 );
}
@ -373,6 +650,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
sqlite3Fts5IterClose(pTerm->pIter);
}
}
fts5BufferFree(&pPhrase->poslist);
sqlite3_free(pPhrase);
}
}
@ -455,13 +733,13 @@ static int fts5ParseTokenize(
sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
);
if( pNew==0 ) return SQLITE_NOMEM;
if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
pCtx->pPhrase = pPhrase = pNew;
pNew->nTerm = nNew - SZALLOC;
}
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
pTerm->bPrefix = 0;
pTerm->pIter = 0;
memset(pTerm, 0, sizeof(Fts5ExprTerm));
pTerm->zTerm = fts5Strdup(pToken, nToken);
return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;

189
ext/fts5/fts5_index.c
View File

@ -254,7 +254,6 @@ static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
typedef struct Fts5BtreeIter Fts5BtreeIter;
typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
typedef struct Fts5Buffer Fts5Buffer;
typedef struct Fts5ChunkIter Fts5ChunkIter;
typedef struct Fts5Data Fts5Data;
typedef struct Fts5MultiSegIter Fts5MultiSegIter;
@ -298,15 +297,6 @@ struct Fts5Index {
sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
};
/*
** Buffer object for the incremental building of string data.
*/
struct Fts5Buffer {
u8 *p;
int n;
int nSpace;
};
struct Fts5IndexIter {
Fts5Index *pIndex;
Fts5Structure *pStruct;
@ -559,129 +549,6 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){
}
static int fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){
/* A no-op if an error has already occurred */
if( *pRc ) return 1;
if( (pBuf->n + nByte) > pBuf->nSpace ){
u8 *pNew;
int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
while( nNew<(pBuf->n + nByte) ){
nNew = nNew * 2;
}
pNew = sqlite3_realloc(pBuf->p, nNew);
if( pNew==0 ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
pBuf->nSpace = nNew;
pBuf->p = pNew;
}
}
return 0;
}
/*
** Encode value iVal as an SQLite varint and append it to the buffer object
** pBuf. If an OOM error occurs, set the error code in p.
*/
static void fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
if( fts5BufferGrow(pRc, pBuf, 9) ) return;
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal);
}
/*
** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
static void fts5BufferAppendBlob(
int *pRc,
Fts5Buffer *pBuf,
int nData,
const u8 *pData
){
if( fts5BufferGrow(pRc, pBuf, nData) ) return;
memcpy(&pBuf->p[pBuf->n], pData, nData);
pBuf->n += nData;
}
/*
** Append the nul-terminated string zStr to the buffer pBuf. This function
** ensures that the byte following the buffer data is set to 0x00, even
** though this byte is not included in the pBuf->n count.
*/
static void fts5BufferAppendString(
int *pRc,
Fts5Buffer *pBuf,
const char *zStr
){
int nStr = strlen(zStr);
if( fts5BufferGrow(pRc, pBuf, nStr+1) ) return;
fts5BufferAppendBlob(pRc, pBuf, nStr, (const u8*)zStr);
if( *pRc==SQLITE_OK ) pBuf->p[pBuf->n] = 0x00;
}
/*
** Argument zFmt is a printf() style format string. This function performs
** the printf() style processing, then appends the results to buffer pBuf.
**
** Like fts5BufferAppendString(), this function ensures that the byte
** following the buffer data is set to 0x00, even though this byte is not
** included in the pBuf->n count.
*/
static void fts5BufferAppendPrintf(
int *pRc,
Fts5Buffer *pBuf,
char *zFmt, ...
){
if( *pRc==SQLITE_OK ){
char *zTmp;
va_list ap;
va_start(ap, zFmt);
zTmp = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zTmp==0 ){
*pRc = SQLITE_NOMEM;
}else{
fts5BufferAppendString(pRc, pBuf, zTmp);
sqlite3_free(zTmp);
}
}
}
/*
** Free any buffer allocated by pBuf. Zero the structure before returning.
*/
static void fts5BufferFree(Fts5Buffer *pBuf){
sqlite3_free(pBuf->p);
memset(pBuf, 0, sizeof(Fts5Buffer));
}
/*
** Zero the contents of the buffer object. But do not free the associated
** memory allocation.
*/
static void fts5BufferZero(Fts5Buffer *pBuf){
pBuf->n = 0;
}
/*
** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
static void fts5BufferSet(
int *pRc,
Fts5Buffer *pBuf,
int nData,
const u8 *pData
){
pBuf->n = 0;
fts5BufferAppendBlob(pRc, pBuf, nData, pData);
}
/*
** Compare the contents of the pLeft buffer with the pRight/nRight blob.
**
@ -1621,6 +1488,11 @@ static void fts5ChunkIterInit(
}
}
static void fts5ChunkIterRelease(Fts5ChunkIter *pIter){
fts5DataRelease(pIter->pLeaf);
pIter->pLeaf = 0;
}
/*
** Read and return the next 32-bit varint from the position-list iterator
** passed as the second argument.
@ -2935,15 +2807,17 @@ static void fts5DecodeStructure(
for(iLvl=0; iLvl<p->nLevel; iLvl++){
Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
fts5BufferAppendPrintf(pRc, pBuf, " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge);
sqlite3Fts5BufferAppendPrintf(pRc, pBuf,
" {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
);
for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
fts5BufferAppendPrintf(pRc, pBuf,
sqlite3Fts5BufferAppendPrintf(pRc, pBuf,
" {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight,
pSeg->pgnoFirst, pSeg->pgnoLast
);
}
fts5BufferAppendPrintf(pRc, pBuf, "}");
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
}
fts5StructureRelease(p);
@ -2984,7 +2858,7 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
while( iOff<n ){
int iVal;
iOff += getVarint32(&a[iOff], iVal);
fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
}
return iOff;
}
@ -3003,7 +2877,7 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
if( iOff<n ){
iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
}
while( iOff<n ){
int nPos;
@ -3014,7 +2888,7 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
if( iDelta==0 ) return iOff;
iDocid -= iDelta;
fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
}
}
@ -3044,16 +2918,18 @@ static void fts5DecodeFunction(
if( iSegid==0 ){
if( iRowid==FTS5_AVERAGES_ROWID ){
fts5BufferAppendPrintf(&rc, &s, "{averages} ");
sqlite3Fts5BufferAppendPrintf(&rc, &s, "{averages} ");
}else{
fts5BufferAppendPrintf(&rc, &s, "{structure idx=%d}", (int)(iRowid-10));
sqlite3Fts5BufferAppendPrintf(&rc, &s,
"{structure idx=%d}", (int)(iRowid-10)
);
fts5DecodeStructure(&rc, &s, a, n);
}
}else{
Fts5Buffer term;
memset(&term, 0, sizeof(Fts5Buffer));
fts5BufferAppendPrintf(&rc, &s, "(idx=%d segid=%d h=%d pgno=%d) ",
sqlite3Fts5BufferAppendPrintf(&rc, &s, "(idx=%d segid=%d h=%d pgno=%d) ",
iIdx, iSegid, iHeight, iPgno
);
@ -3089,7 +2965,7 @@ static void fts5DecodeFunction(
fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
iOff += nByte;
fts5BufferAppendPrintf(
sqlite3Fts5BufferAppendPrintf(
&rc, &s, " term=%.*s", term.n, (const char*)term.p
);
iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
@ -3102,12 +2978,14 @@ static void fts5DecodeFunction(
Fts5NodeIter ss;
for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){
if( ss.term.n==0 ){
fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
}else{
fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", ss.term.n, ss.term.p);
sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"",
ss.term.n, ss.term.p
);
}
if( ss.nEmpty ){
fts5BufferAppendPrintf(&rc, &s, " empty=%d", ss.nEmpty);
sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d", ss.nEmpty);
}
}
fts5NodeIterFree(&ss);
@ -3169,6 +3047,7 @@ Fts5IndexIter *sqlite3Fts5IndexQuery(
pRet = (Fts5IndexIter*)sqlite3_malloc(sizeof(Fts5IndexIter));
if( pRet ){
memset(pRet, 0, sizeof(Fts5IndexIter));
pRet->pStruct = fts5StructureRead(p, 0);
if( pRet->pStruct ){
fts5MultiIterNew(p,
@ -3216,8 +3095,23 @@ i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
** disk.
*/
const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, int *pn){
assert( sqlite3Fts5IterEof(pIter)==0 );
Fts5ChunkIter iter;
Fts5Index *p = pIter->pIndex;
Fts5SegIter *pSeg = &pIter->pMulti->aSeg[ pIter->pMulti->aFirst[1] ];
assert( sqlite3Fts5IterEof(pIter)==0 );
fts5ChunkIterInit(p, pSeg, &iter);
if( fts5ChunkIterEof(p, &iter)==0 ){
fts5BufferZero(&pIter->poslist);
fts5BufferGrow(&p->rc, &pIter->poslist, iter.nRem);
while( fts5ChunkIterEof(p, &iter)==0 ){
fts5BufferAppendBlob(&p->rc, &pIter->poslist, iter.n, iter.p);
fts5ChunkIterNext(p, &iter);
}
}
fts5ChunkIterRelease(&iter);
if( p->rc ) return 0;
*pn = pIter->poslist.n;
return pIter->poslist.p;
}
@ -3230,6 +3124,7 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
fts5MultiIterFree(pIter->pIndex, pIter->pMulti);
fts5StructureRelease(pIter->pStruct);
fts5CloseReader(pIter->pIndex);
fts5BufferFree(&pIter->poslist);
sqlite3_free(pIter);
}
}

4
main.mk
View File

@ -73,6 +73,7 @@ LIBOBJ+= vdbe.o parse.o \
vdbetrace.o wal.o walker.o where.o utf.o vtab.o
LIBOBJ += fts5.o
LIBOBJ += fts5_buffer.o
LIBOBJ += fts5_config.o
LIBOBJ += fts5_expr.o
LIBOBJ += fts5_index.o
@ -572,6 +573,9 @@ rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
# FTS5 things
#
fts5_buffer.o: $(TOP)/ext/fts5/fts5_buffer.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_buffer.c
fts5_config.o: $(TOP)/ext/fts5/fts5_config.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_config.c

21
manifest
View File

@ -1,5 +1,5 @@
C Change\sthe\sposition\slist\sformat\sso\sthat\sits\ssize\sin\sbytes\sis\sstored\sat\sthe\sstart\sof\sthe\slist\sitself.
D 2014-07-01T20:45:18.496
C Add\ssupport\sfor\sphrase\squeries\sto\sfts5.
D 2014-07-02T20:18:49.027
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -104,10 +104,11 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
F ext/fts5/fts5.c 1af3184dd9c0e5c1686f71202d6b6cac8f225f05
F ext/fts5/fts5Int.h 80f3d38a69a0c58ccc94428c8fc8adbcf7561a2d
F ext/fts5/fts5Int.h b7a684ff3508ab24437886f8bc873a16f494a7db
F ext/fts5/fts5_buffer.c f1a26a79e2943fe4388e531fa141941b5eb6d31a
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
F ext/fts5/fts5_expr.c 1874b17f10a38d0b21e0c38a28637f74e4d2570a
F ext/fts5/fts5_index.c ea3dfe56a16813fcf59e03f6156965894b4b5e6f
F ext/fts5/fts5_expr.c aacfcf6b7c14ca5987ba1de0bd080eee31fca98c
F ext/fts5/fts5_index.c 6bb95f6a1ed0e50bc9f2dce7b7a92859f5821364
F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
@ -152,7 +153,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F main.mk 2bb1ec703ac4f27743961764b59cfb5f91d72bfe
F main.mk c5524f888196af43a9b5dfae878205044f549dbf
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@ -592,7 +593,7 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343
F test/fts5ab.test 6436ad345d1e7eb5ab198c0174834380805f609c
F test/fts5ab.test bdc1dd9d58163c0c7b184be817f82e3bf8a81c37
F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@ -1188,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 94eeb077d08a1d2607f3ff3a9fbf18229ba475bb
R 5d9b8f6933c58725a24e426a963b0d97
P 62f2ff20418702ed0fbf708369edf5638445b51b
R 773d748328905c117f50682aca9f537a
U dan
Z bb8816e0d501865bff7c4c8da87350cb
Z 194302280b1431e713a39a0adc2c19fe

2
manifest.uuid
View File

@ -1 +1 @@
62f2ff20418702ed0fbf708369edf5638445b51b
2e5652e6526b8fb3f5c163168d95bc0bb4c93686

43
test/fts5ab.test
View File

@ -94,11 +94,42 @@ foreach {tn expr res} {
do_execsql_test 2.7.$tn { SELECT rowid FROM t1 WHERE t1 MATCH $expr } $res
}
#db eval {
# SELECT fts5_decode(rowid, block) AS t FROM t1_data;
#} {
# puts $t
#}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a,b);
INSERT INTO t1(t1) VALUES('pgsz=32');
}
foreach {tn a b} {
1 {abashed abandons abase abash abaft} {abases abased}
2 {abasing abases abaft abated abandons} {abases abandoned}
3 {abatement abash abash abated abase} {abasements abashing}
4 {abaft abasements abase abasement abasing} {abasement abases}
5 {abaft abashing abatement abash abasements} {abandons abandoning}
6 {aback abate abasements abashes abandoned} {abasement abased}
7 {abandons abated abased aback abandoning} {abases abandoned}
8 {abashing abases abasement abaft abashing} {abashed abate}
9 {abash abase abate abashing abashed} {abandon abandoned}
10 {abate abandoning abandons abasement aback} {abandon abandoning}
} {
do_execsql_test 2.1.$tn.1 { INSERT INTO t1 VALUES($a, $b) }
do_execsql_test 2.1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach {tn expr res} {
1 {abash} {9 5 3 1}
2 {abase} {9 4 3 1}
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {8 5}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
do_execsql_test 2.2.$tn {
SELECT rowid FROM t1 WHERE t1 MATCH $expr
} $res
}
finish_test