mirrors
/
sqlite
mirror of https://github.com/sqlite/sqlite
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add missing comments and fix compiler warnings in new FTS3/4 code. Other minor fixes too. 

FossilOrigin-Name: 1c9c70fec3c88319f7b2efe5316694a6ce0ab1a5
This commit is contained in:
dan 2010-10-22 16:44:39 +00:00
parent ca3b8f9c86
commit 789cb8f5cf
6 changed files with 294 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

313
ext/fts3/fts3.c
View File

@ -970,7 +970,21 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
}
}
/*
** This function is used to process a single interior node when searching
** a b-tree for a term or term prefix. The node data is passed to this
** function via the zNode/nNode parameters. The term to search for is
** passed in zTerm/nTerm.
**
** If piFirst is not NULL, then this function sets *piFirst to the blockid
** of the child node that heads the sub-tree that may contain the term.
**
** If piLast is not NULL, then *piLast is set to the right-most child node
** that heads a sub-tree that may contain a term for which zTerm/nTerm is
** a prefix.
**
** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
*/
static int fts3ScanInteriorNode(
Fts3Table *p, /* Virtual table handle */
const char *zTerm, /* Term to select leaves for */
@ -985,22 +999,23 @@ static int fts3ScanInteriorNode(
const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
char *zBuffer = 0; /* Buffer to load terms into */
int nAlloc = 0; /* Size of allocated buffer */
int isFirstTerm = 1; /* True when processing first term on page */
sqlite3_int64 iChild; /* Block id of child node to descend to */
int isFirstTerm = 1; /* True when processing first term on page */
int dummy;
sqlite3_int64 iChild; /* Block id of child node to descend to */
int nBlock; /* Size of child node in bytes */
zCsr += sqlite3Fts3GetVarint32(zCsr, &dummy);
/* Skip over the 'height' varint that occurs at the start of every
** interior node. Then load the blockid of the left-child of the b-tree
** node into variable iChild. */
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
while( zCsr<zEnd && (piFirst || piLast) ){
int cmp; /* memcmp() result */
int nSuffix; /* Size of term suffix */
int nPrefix = 0; /* Size of term prefix */
int nBuffer; /* Total term size */
int cmp; /* memcmp() result */
int nSuffix; /* Size of term suffix */
int nPrefix = 0; /* Size of term prefix */
int nBuffer; /* Total term size */
/* Load the next term on the node into zBuffer */
/* Load the next term on the node into zBuffer. Use realloc() to expand
** the size of zBuffer if required. */
if( !isFirstTerm ){
zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
}
@ -1052,18 +1067,23 @@ static int fts3ScanInteriorNode(
/*
** The buffer pointed to by argument zNode (size nNode bytes) contains the
** root node of a b-tree segment. The segment is guaranteed to be at least
** one level high (i.e. the root node is not also a leaf). If successful,
** this function locates the leaf node of the segment that may contain the
** term specified by arguments zTerm and nTerm and writes its block number
** to *piLeaf.
** The buffer pointed to by argument zNode (size nNode bytes) contains an
** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
** contains a term. This function searches the sub-tree headed by the zNode
** node for the range of leaf nodes that may contain the specified term
** or terms for which the specified term is a prefix.
**
** It is possible that the returned leaf node does not contain the specified
** term. However, if the segment does contain said term, it is stored on
** the identified leaf node. Because this function only inspects interior
** segment nodes (and never loads leaf nodes into memory), it is not possible
** to be sure.
** If piLeaf is not NULL, then *piLeaf is set to the blockid of the
** left-most leaf node in the tree that may contain the specified term.
** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
** right-most leaf node that may contain a term for which the specified
** term is a prefix.
**
** It is possible that the range of returned leaf nodes does not contain
** the specified term or any terms for which it is a prefix. However, if the
** segment does contain any such terms, they are stored within the identified
** range. Because this function only inspects interior segment nodes (and
** never loads leaf nodes into memory), it is not possible to be sure.
**
** If an error occurs, an error code other than SQLITE_OK is returned.
*/
@ -1079,25 +1099,30 @@ static int fts3SelectLeaf(
int rc; /* Return code */
int iHeight; /* Height of this node in tree */
assert( piLeaf || piLeaf2 );
sqlite3Fts3GetVarint32(zNode, &iHeight);
rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
if( rc==SQLITE_OK && iHeight>1 ){
const char *zBlob;
int nBlob;
char *zBlob = 0; /* Blob read from %_segments table */
int nBlob; /* Size of zBlob in bytes */
if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
if( rc==SQLITE_OK ){
rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
}
sqlite3_free(zBlob);
piLeaf = 0;
zBlob = 0;
}
rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
if( rc==SQLITE_OK ){
rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
}
sqlite3_free(zBlob);
}
return rc;
@ -1946,38 +1971,24 @@ static int fts3TermSegReaderArray(
*/
rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
}else{
int rc2; /* Return value of sqlite3Fts3ReadBlock() */
sqlite3_int64 i1; /* First leaf that may contain zTerm */
sqlite3_int64 i2; /* Last leaf that may contain zTerm */
sqlite3_int64 i2; /* Final leaf that may contain zTerm */
rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
if( isPrefix==0 ) i2 = i1;
if( rc==SQLITE_OK ){
rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
}
/* The following call to ReadBlock() serves to reset the SQL statement
** used to retrieve blocks of data from the %_segments table. If it is
** not reset here, then it may remain classified as an active statement
** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands
** failing.
*/
rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
if( rc==SQLITE_OK ){
rc = rc2;
}
}
iAge++;
assert( (pNew==0)==(rc!=SQLITE_OK) );
/* If a new Fts3SegReader was allocated, add it to the array. */
assert( pNew!=0 || rc!=SQLITE_OK );
if( rc==SQLITE_OK ){
rc = fts3SegReaderArrayAdd(&pArray, pNew);
}else{
sqlite3Fts3SegReaderFree(p, pNew);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost);
}
iAge++;
}
if( rc==SQLITE_DONE ){
@ -2109,12 +2120,21 @@ static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
return rc;
}
static void fts3DoclistStripPositions(char *aList, int *pnList){
/*
** This function removes the position information from a doclist. When
** called, buffer aList (size *pnList bytes) contains a doclist that includes
** position information. This function removes the position information so
** that aList contains only docids, and adjusts *pnList to reflect the new
** (possibly reduced) size of the doclist.
*/
static void fts3DoclistStripPositions(
char *aList, /* IN/OUT: Buffer containing doclist */
int *pnList /* IN/OUT: Size of doclist in bytes */
){
if( aList ){
char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
char *p = aList; /* Input cursor */
char *pOut = aList; /* Output cursor */
sqlite3_int64 iPrev = 0;
while( p<aEnd ){
sqlite3_int64 delta;
@ -2274,6 +2294,14 @@ static int fts3PhraseSelect(
return rc;
}
/*
** This function merges two doclists according to the requirements of a
** NEAR operator.
**
** Both input doclists must include position information. The output doclist
** includes position information if the first argument to this function
** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
*/
static int fts3NearMerge(
int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
int nNear, /* Parameter to NEAR operator */
@ -2286,8 +2314,8 @@ static int fts3NearMerge(
char **paOut, /* OUT: Results of merge (malloced) */
int *pnOut /* OUT: Sized of output buffer */
){
char *aOut;
int rc;
char *aOut; /* Buffer to write output doclist to */
int rc; /* Return code */
assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
@ -2308,8 +2336,23 @@ static int fts3NearMerge(
return rc;
}
/*
** This function is used as part of the processing for the snippet() and
** offsets() functions.
**
** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
** have their respective doclists (including position information) loaded
** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
** each doclist that are not within nNear tokens of a corresponding entry
** in the other doclist.
*/
int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
int rc;
int rc; /* Return code */
assert( pLeft->eType==FTSQUERY_PHRASE );
assert( pRight->eType==FTSQUERY_PHRASE );
assert( pLeft->isLoaded && pRight->isLoaded );
if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
sqlite3_free(pLeft->aDoclist);
sqlite3_free(pRight->aDoclist);
@ -2317,8 +2360,8 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
pLeft->aDoclist = 0;
rc = SQLITE_OK;
}else{
char *aOut;
int nOut;
char *aOut; /* Buffer in which to assemble new doclist */
int nOut; /* Size of buffer aOut in bytes */
rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
@ -2342,41 +2385,12 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
return rc;
}
typedef struct ExprAndCost ExprAndCost;
struct ExprAndCost {
Fts3Expr *pExpr;
int nCost;
};
int fts3ExprCost(Fts3Expr *pExpr){
int nCost; /* Return value */
if( pExpr->eType==FTSQUERY_PHRASE ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int ii;
nCost = 0;
for(ii=0; ii<pPhrase->nToken; ii++){
nCost += pPhrase->aToken[ii].pArray->nCost;
}
}else{
nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
}
return nCost;
}
static void fts3ExprAssignCosts(
Fts3Expr *pExpr, /* Expression to create seg-readers for */
ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
){
if( pExpr->eType==FTSQUERY_AND ){
fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
}else{
(*ppExprCost)->pExpr = pExpr;
(*ppExprCost)->nCost = fts3ExprCost(pExpr);;
(*ppExprCost)++;
}
}
/*
** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
** The allocated objects are stored in the Fts3PhraseToken.pArray member
** variables of each token structure.
*/
static int fts3ExprAllocateSegReaders(
Fts3Cursor *pCsr, /* FTS3 table */
Fts3Expr *pExpr, /* Expression to create seg-readers for */
@ -2391,7 +2405,6 @@ static int fts3ExprAllocateSegReaders(
}
if( pExpr->eType==FTSQUERY_PHRASE ){
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
Fts3Phrase *pPhrase = pExpr->pPhrase;
int ii;
@ -2412,6 +2425,11 @@ static int fts3ExprAllocateSegReaders(
return rc;
}
/*
** Free the Fts3SegReaderArray objects associated with each token in the
** expression pExpr. In other words, this function frees the resources
** allocated by fts3ExprAllocateSegReaders().
*/
static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
if( pExpr ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
@ -2428,10 +2446,89 @@ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
}
/*
** Evaluate the full-text expression pExpr against fts3 table pTab. Store
** the resulting doclist in *paOut and *pnOut. This routine mallocs for
** the space needed to store the output. The caller is responsible for
** Return the sum of the costs of all tokens in the expression pExpr. This
** function must be called after Fts3SegReaderArrays have been allocated
** for all tokens using fts3ExprAllocateSegReaders().
*/
int fts3ExprCost(Fts3Expr *pExpr){
int nCost; /* Return value */
if( pExpr->eType==FTSQUERY_PHRASE ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int ii;
nCost = 0;
for(ii=0; ii<pPhrase->nToken; ii++){
nCost += pPhrase->aToken[ii].pArray->nCost;
}
}else{
nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
}
return nCost;
}
/*
** The following is a helper function (and type) for fts3EvalExpr(). It
** must be called after Fts3SegReaders have been allocated for every token
** in the expression. See the context it is called from in fts3EvalExpr()
** for further explanation.
*/
typedef struct ExprAndCost ExprAndCost;
struct ExprAndCost {
Fts3Expr *pExpr;
int nCost;
};
static void fts3ExprAssignCosts(
Fts3Expr *pExpr, /* Expression to create seg-readers for */
ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
){
if( pExpr->eType==FTSQUERY_AND ){
fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
}else{
(*ppExprCost)->pExpr = pExpr;
(*ppExprCost)->nCost = fts3ExprCost(pExpr);;
(*ppExprCost)++;
}
}
/*
** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
** the resulting doclist in *paOut and *pnOut. This routine mallocs for
** the space needed to store the output. The caller is responsible for
** freeing the space when it has finished.
**
** This function is called in two distinct contexts:
**
** * From within the virtual table xFilter() method. In this case, the
** output doclist contains entries for all rows in the table, based on
** data read from the full-text index.
**
** In this case, if the query expression contains one or more tokens that
** are very common, then the returned doclist may contain a superset of
** the documents that actually match the expression.
**
** * From within the virtual table xNext() method. This call is only made
** if the call from within xFilter() found that there were very common
** tokens in the query expression and did return a superset of the
** matching documents. In this case the returned doclist contains only
** entries that correspond to the current row of the table. Instead of
** reading the data for each token from the full-text index, the data is
** already available in-memory in the Fts3PhraseToken.pDeferred structures.
** See fts3EvalDeferred() for how it gets there.
**
** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
** required) Fts3Cursor.doDeferred==1.
**
** If the SQLite invokes the snippet(), offsets() or matchinfo() function
** as part of a SELECT on an FTS3 table, this function is called on each
** individual phrase expression in the query. If there were very common tokens
** found in the xFilter() call, then this function is called once for phrase
** for each row visited, and the returned doclist contains entries for the
** current row only. Otherwise, if there were no very common tokens, then this
** function is called once only for each phrase in the query and the returned
** doclist contains entries for all rows of the table.
**
** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
** result of a snippet(), offsets() or matchinfo() invocation.
*/
static int fts3EvalExpr(
Fts3Cursor *p, /* Virtual table cursor handle */
@ -2596,9 +2693,26 @@ static int fts3EvalExpr(
}
/*
** This function is called from within xNext() for each row visited by
** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
** was able to determine the exact set of matching rows, this function sets
** *pbRes to true and returns SQLITE_IO immediately.
**
** Otherwise, if evaluating the query expression within xFilter() returned a
** superset of the matching documents instead of an exact set (this happens
** when the query includes very common tokens and it is deemed too expensive to
** load their doclists from disk), this function tests if the current row
** really does match the FTS3 query.
**
** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
** is returned and *pbRes is set to true if the current row matches the
** FTS3 query (and should be included in the results returned to SQLite), or
** false otherwise.
*/
static int fts3EvalDeferred(Fts3Cursor *pCsr, int *pbRes){
static int fts3EvalDeferred(
Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
int *pbRes /* OUT: Set to true if row is a match */
){
int rc = SQLITE_OK;
if( pCsr->pDeferred==0 ){
*pbRes = 1;
@ -2677,11 +2791,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
** side of the MATCH operator.
*/
/* TODO(shess) Upgrade the cursor initialization and destruction to
** account for fts3FilterMethod() being called multiple times on the
** same cursor. The current solution is very fragile. Apply fix to
** fts3 as appropriate.
*/
static int fts3FilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
@ -2785,6 +2894,11 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
if( pCsr->aDoclist ){
*pRowid = pCsr->iPrevId;
}else{
/* This branch runs if the query is implemented using a full-table scan
** (not using the full-text index). In this case grab the rowid from the
** SELECT statement.
*/
assert( pCsr->isRequireSeek==0 );
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}
return SQLITE_OK;
@ -3210,19 +3324,20 @@ static void hashDestroy(void *p){
}
/*
** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
** two forward declarations are for functions declared in these files
** used to retrieve the respective implementations.
** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are
** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
** respectively. The following three forward declarations are for functions
** declared in these files used to retrieve the respective implementations.
**
** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
** to by the argument to point to the "simple" tokenizer implementation.
** Function ...PorterTokenizerModule() sets *pModule to point to the
** porter tokenizer/stemmer implementation.
** And so on.
*/
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#ifdef SQLITE_ENABLE_ICU
void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#endif
/*
** Initialise the fts3 extension. If this extension is built as part

20
ext/fts3/fts3Int.h
View File

@ -122,13 +122,13 @@ struct Fts3Table {
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
sqlite3_stmt *aStmt[25];
sqlite3_stmt *aStmt[24];
char *zSegmentsTbl; /* Name of %_segments table */
int nPgsz; /* Page size for host database */
int nNodeSize; /* Soft limit for node size */
u8 bHasContent; /* True if %_content table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
int nPgsz; /* Page size for host database */
char *zSegmentsTbl; /* Name of %_segments table */
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
/* The following hash table is used to buffer pending index updates during
@ -163,7 +163,6 @@ struct Fts3Cursor {
int nDoclist; /* Size of buffer at aDoclist */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
int doDeferred;
int nRowAvg; /* Average size of database rows, in pages */
};
@ -193,13 +192,12 @@ struct Fts3Cursor {
** For a sequence of tokens contained in double-quotes (i.e. "one two three")
** nToken will be the number of tokens in the string.
*/
struct Fts3PhraseToken {
char *z; /* Text of the token */
int n; /* Number of bytes in buffer z */
int isPrefix; /* True if token ends with a "*" character */
Fts3SegReaderArray *pArray;
Fts3DeferredToken *pDeferred;
Fts3SegReaderArray *pArray; /* Segment-reader for this token */
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
};
struct Fts3Phrase {
@ -258,11 +256,6 @@ struct Fts3Expr {
#define FTSQUERY_PHRASE 5
/* fts3_init.c */
int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*,
sqlite3_vtab **, char **);
/* fts3_write.c */
int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
int sqlite3Fts3PendingTermsFlush(Fts3Table *);
@ -276,11 +269,12 @@ int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
int (*)(Fts3Table *, void *, char *, int, char *, int), void *
);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);

136
ext/fts3/fts3_write.c
View File

@ -78,11 +78,9 @@ struct Fts3SegReader {
sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
sqlite3_blob *pBlob; /* Blob open on iStartBlock */
char *aNode; /* Pointer to node data (or NULL) */
int nNode; /* Size of buffer at aNode (or 0) */
int nTermAlloc; /* Allocated size of zTerm buffer */
Fts3HashElem **ppNextElem;
/* Variables set by fts3SegReaderNext(). These may be read directly
@ -92,6 +90,7 @@ struct Fts3SegReader {
*/
int nTerm; /* Number of bytes in current term */
char *zTerm; /* Pointer to current term */
int nTermAlloc; /* Allocated size of zTerm buffer */
char *aDoclist; /* Pointer to doclist of current entry */
int nDoclist; /* Size of doclist in current entry */
@ -170,12 +169,11 @@ struct SegmentNode {
#define SQL_DELETE_SEGDIR_BY_LEVEL 16
#define SQL_DELETE_SEGMENTS_RANGE 17
#define SQL_CONTENT_INSERT 18
#define SQL_GET_BLOCK 19
#define SQL_DELETE_DOCSIZE 20
#define SQL_REPLACE_DOCSIZE 21
#define SQL_SELECT_DOCSIZE 22
#define SQL_SELECT_DOCTOTAL 23
#define SQL_REPLACE_DOCTOTAL 24
#define SQL_DELETE_DOCSIZE 19
#define SQL_REPLACE_DOCSIZE 20
#define SQL_SELECT_DOCSIZE 21
#define SQL_SELECT_DOCTOTAL 22
#define SQL_REPLACE_DOCTOTAL 23
/*
** This function is used to obtain an SQLite prepared statement handle
@ -220,12 +218,11 @@ static int fts3SqlStmt(
/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
@ -300,45 +297,6 @@ static void fts3SqlExec(
}
/*
** Read a single block from the %_segments table. If the specified block
** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
** etc.) occurs, return the appropriate SQLite error code.
**
** Otherwise, if successful, set *pzBlock to point to a buffer containing
** the block read from the database, and *pnBlock to the size of the read
** block in bytes.
**
** WARNING: The returned buffer is only valid until the next call to
** sqlite3Fts3ReadBlock().
*/
int sqlite3Fts3ReadBlock(
Fts3Table *p,
sqlite3_int64 iBlock,
char const **pzBlock,
int *pnBlock
){
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_reset(pStmt);
if( pzBlock ){
sqlite3_bind_int64(pStmt, 1, iBlock);
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW ){
return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
}
*pnBlock = sqlite3_column_bytes(pStmt, 0);
*pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
return SQLITE_CORRUPT;
}
}
return SQLITE_OK;
}
/*
** This function ensures that the caller has obtained a shared-cache
** table-lock on the %_content table. This is required before reading
@ -595,6 +553,9 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
return SQLITE_OK;
}
/*
** Discard the contents of the pending-terms hash table.
*/
void sqlite3Fts3PendingTermsClear(Fts3Table *p){
Fts3HashElem *pElem;
for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
@ -807,24 +768,49 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
** The %_segments table is declared as follows:
**
** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
**
** This function reads data from a single row of the %_segments table. The
** specific row is identified by the iBlockid parameter. If paBlob is not
** NULL, then a buffer is allocated using sqlite3_malloc() and populated
** with the contents of the blob stored in the "block" column of the
** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
** to the size of the blob in bytes before returning.
**
** If an error occurs, or the table does not contain the specified row,
** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
** paBlob is non-NULL, then it is the responsibility of the caller to
** eventually free the returned buffer.
**
** This function may leave an open sqlite3_blob* handle in the
** Fts3Table.pSegments variable. This handle is reused by subsequent calls
** to this function. The handle may be closed by calling the
** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
** performance improvement, but the blob handle should always be closed
** before control is returned to the user (to prevent a lock being held
** on the database file for longer than necessary). Thus, any virtual table
** method (xFilter etc.) that may directly or indirectly call this function
** must call sqlite3Fts3SegmentsClose() before returning.
*/
static int fts3SegmentsBlob(
Fts3Table *p,
sqlite3_int64 iSegment,
char **paBlob,
int *pnBlob
int sqlite3Fts3ReadBlock(
Fts3Table *p, /* FTS3 table handle */
sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
char **paBlob, /* OUT: Blob data in malloc'd buffer */
int *pnBlob /* OUT: Size of blob data */
){
int rc;
int rc; /* Return code */
/* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
assert( pnBlob);
if( p->pSegments ){
rc = sqlite3_blob_reopen(p->pSegments, iSegment);
rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
}else{
if( 0==p->zSegmentsTbl ){
p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
}
rc = sqlite3_blob_open(
p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, &p->pSegments
p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
);
}
@ -849,12 +835,15 @@ static int fts3SegmentsBlob(
return rc;
}
/*
** Close the blob handle at p->pSegments, if it is open. See comments above
** the sqlite3Fts3ReadBlock() function for details.
*/
void sqlite3Fts3SegmentsClose(Fts3Table *p){
sqlite3_blob_close(p->pSegments);
p->pSegments = 0;
}
/*
** Move the iterator passed as the first argument to the next term in the
** segment. If successful, SQLITE_OK is returned. If there is no next term,
@ -872,8 +861,7 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
}
if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
sqlite3_blob *pBlob;
int rc;
int rc; /* Return code from Fts3ReadBlock() */
if( fts3SegReaderIsPending(pReader) ){
Fts3HashElem *pElem = *(pReader->ppNextElem);
@ -898,17 +886,15 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
/* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
** blocks have already been traversed. */
assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
return SQLITE_OK;
}
rc = fts3SegmentsBlob(
rc = sqlite3Fts3ReadBlock(
p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
);
if( rc!=SQLITE_OK ){
return rc;
}
if( rc!=SQLITE_OK ) return rc;
pNext = pReader->aNode;
}
@ -1011,7 +997,6 @@ int sqlite3Fts3SegReaderCost(
Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
int rc = SQLITE_OK; /* Return code */
int nCost = 0; /* Cost in bytes to return */
sqlite3_int64 iLeaf; /* Used to iterate through required leaves */
int pgsz = p->nPgsz; /* Database page size */
/* If this seg-reader is reading the pending-terms table, or if all data
@ -1065,7 +1050,7 @@ int sqlite3Fts3SegReaderCost(
** confirms this).
*/
for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
rc = fts3SegmentsBlob(p, iBlock, 0, &nBlob);
rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
if( rc!=SQLITE_OK ) break;
if( (nBlob+35)>pgsz ){
int nOvfl = (nBlob + 34)/pgsz;
@ -1134,7 +1119,6 @@ int sqlite3Fts3SegReaderNew(
}
rc = fts3SegReaderNext(p, pReader);
finished:
if( rc==SQLITE_OK ){
*ppReader = pReader;
}else{
@ -1471,7 +1455,7 @@ static int fts3PrefixCompress(
** (according to memcmp) than the previous term.
*/
static int fts3NodeAddTerm(
Fts3Table *p, /* Virtual table handle */
Fts3Table *p, /* Virtual table handle */
SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
int isCopyTerm, /* True if zTerm/nTerm is transient */
const char *zTerm, /* Pointer to buffer containing term */
@ -2244,7 +2228,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
int i; /* Iterator variable */
int rc; /* Return code */
int iIdx; /* Index of new segment */
int iNewLevel; /* Level to create new segment at */
int iNewLevel = 0; /* Level to create new segment at */
sqlite3_stmt *pStmt = 0;
SegmentWriter *pWriter = 0;
int nSegment = 0; /* Number of segments being merged */
@ -2722,7 +2706,7 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
const char *zText = sqlite3_column_text(pCsr->pStmt, i+1);
const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
sqlite3_tokenizer_cursor *pTC = 0;
rc = pModule->xOpen(pT, zText, -1, &pTC);
@ -2812,7 +2796,7 @@ int sqlite3Fts3UpdateMethod(
/* If this is a DELETE or UPDATE operation, remove the old record. */
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
int isEmpty;
int isEmpty = 0;
rc = fts3IsEmpty(p, apVal, &isEmpty);
if( rc==SQLITE_OK ){
if( isEmpty ){

18
manifest
View File

@ -1,5 +1,5 @@
C Merge\strunk\schanges\sinto\sexperimental\sbranch.
D 2010-10-21T15:49:47
C Add\smissing\scomments\sand\sfix\scompiler\swarnings\sin\snew\sFTS3/4\scode.\sOther\sminor\sfixes\stoo.
D 2010-10-22T16:44:39
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2c8cefd962eca0147132c7cf9eaa4bb24c656f3f
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c f423181b76fc35c38c4dcf4d8aac012813817f34
F ext/fts3/fts3.c e2f031ea6b213371a31cc5bf181c2177fef86aad
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h f80be5abfb24e51cb816287230c0d0c8f1712f59
F ext/fts3/fts3Int.h 068d80157cc7a4bf674d2df817f3b427001ad94a
F ext/fts3/fts3_expr.c a5aee50edde20e5c9116199bd58be869a3a22c9f
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
@ -73,7 +73,7 @@ F ext/fts3/fts3_snippet.c ca60a2a47de5e7abb22a804ccd1a743f81c2fe3e
F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c be47d30cf80bc91e050ece18e2de7e207432be1a
F ext/fts3/fts3_write.c 54ddeed7323f62af6e55162f0d4102822b991684
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -431,7 +431,7 @@ F test/fts3ao.test 8fee868a0e131b98ce3e8907dc69936278e8b29a
F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
F test/fts3cov.test 6f1ff88ff6b5abcfff6979098cb9d0c68a69202e
F test/fts3cov.test 54cf1f98c72abee246447cd688590898c9ecbaf7
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3defer.test cf66bf69afcc2fb8373d3aed31c55399409e83f2
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
@ -875,7 +875,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P d0a450ce78e99f55c862f26f9332786660007a0a f91471e7234db490f97298b1ccb8d6c7fc45b089
R 3c9178950d1e4be3aecf1d9f3dffa25a
P fd1e5cade04961c2f5438a1dfcc2e15eafb4503f
R 206dc3599103f123f01ecb7943eabb05
U dan
Z 5bbfebd98739f68617c4d86986bdd88f
Z dd51b6660510db15d5633c2bf486673b

2
manifest.uuid
View File

@ -1 +1 @@
fd1e5cade04961c2f5438a1dfcc2e15eafb4503f
1c9c70fec3c88319f7b2efe5316694a6ce0ab1a5

8
test/fts3cov.test
View File

@ -6,9 +6,7 @@
#
#***********************************************************************
#
# The tests in this file are structural coverage tests. They are designed
# to complement the tests in fts3rnd.test and fts3doc.test. Between them,
# the three files should provide full coverage of the fts3 extension code.
# The tests in this file are structural coverage tests for FTS3.
#
set testdir [file dirname $argv0]
@ -97,7 +95,7 @@ do_test fts3cov-2.2 {
} {}
do_error_test fts3cov-2.3 {
SELECT * FROM t1 WHERE t1 MATCH 'c*'
} {database disk image is malformed}
} {SQL logic error or missing database}
# Test the "replaced with NULL" case:
do_test fts3cov-2.4 {
@ -105,7 +103,7 @@ do_test fts3cov-2.4 {
} {}
do_error_test fts3cov-2.5 {
SELECT * FROM t1 WHERE t1 MATCH 'cloud'
} {database disk image is malformed}
} {SQL logic error or missing database}
#--------------------------------------------------------------------------
# The following tests are to test the effects of OOM errors while storing