diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index afb2699be5..2320d70b7d 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -108,13 +108,15 @@ proc get_tokenizer_docs {data} { append res "
$line

\n" continue } + if {[regexp {SYNONYM SUPPORT} $line]} { + set line "

Synonym Support

" + } if {[string trim $line] == ""} { append res "

\n" } else { append res "$line\n" } } - append res "\n" set res } @@ -208,6 +210,10 @@ proc main {data} { fts5_tokenizer { output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"] + output [get_fts5_struct $data \ + "Flags that may be passed as the third argument to xTokenize()" \ + "#define FTS5_TOKEN_COLOCATED" + ] } fts5_extension { diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c123d6444c..6872918e94 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -217,7 +217,7 @@ struct Fts5ExtensionApi { int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); int (*xPhraseCount)(Fts5Context*); @@ -278,18 +278,46 @@ struct Fts5ExtensionApi { ** ** xTokenize: ** This function is expected to tokenize the nText byte string indicated -** by argument pText. pText may not be nul-terminated. The first argument -** passed to this function is a pointer to an Fts5Tokenizer object returned -** by an earlier call to xCreate(). +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**

** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer -** passed as the second argument to xTokenize(). The next two arguments -** are a pointer to a buffer containing the token text, and the size of -** the token in bytes. The 4th and 5th arguments are the byte offsets of -** the first byte of and first byte immediately following the text from +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from ** which the token is derived within the input. ** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** ** FTS5 assumes the xToken() callback is invoked for each token in the ** order that they occur within the input text. ** @@ -301,6 +329,112 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether is is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; typedef struct fts5_tokenizer fts5_tokenizer; @@ -309,9 +443,11 @@ struct fts5_tokenizer { void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Byte offset of token within input text */ @@ -320,6 +456,16 @@ struct fts5_tokenizer { ); }; +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + /* ** END OF CUSTOM TOKENIZERS *************************************************************************/ @@ -329,7 +475,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 1 */ + int iVersion; /* Currently always set to 2 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5298429437..6f6f4ed784 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -117,6 +117,12 @@ typedef struct Fts5Config Fts5Config; ** bColumnsize: ** True if the %_docsize table is created. ** +** bPrefixIndex: +** This is only used for debugging. If set to false, any prefix indexes +** are ignored. This value is configured using: +** +** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); +** */ struct Fts5Config { sqlite3 *db; /* Database handle */ @@ -145,10 +151,14 @@ struct Fts5Config { /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; + +#ifdef SQLITE_DEBUG + int bPrefixIndex; /* True to use prefix-indexes */ +#endif }; /* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 3 +#define FTS5_CURRENT_VERSION 4 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 @@ -166,9 +176,10 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); @@ -234,8 +245,10 @@ struct Fts5PoslistReader { int n; /* Size of buffer at a[] in bytes */ int i; /* Current offset in a[] */ + u8 bFlag; /* For client use (any custom purpose) */ + /* Output variables */ - int bEof; /* Set to true at EOF */ + u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ }; int sqlite3Fts5PoslistReaderInit( @@ -375,15 +388,9 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit); int sqlite3Fts5IndexRollback(Fts5Index *p); /* -** Retrieve and clear the current error code, respectively. +** Get or set the "averages" values. */ -int sqlite3Fts5IndexErrcode(Fts5Index*); -void sqlite3Fts5IndexReset(Fts5Index*); - -/* -** Get or set the "averages" record. -*/ -int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* @@ -596,7 +603,7 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); -int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**); +int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written @@ -665,17 +672,6 @@ int sqlite3Fts5TokenizerInit(fts5_api*); ** End of interface to code in fts5_tokenizer.c. **************************************************************************/ -/************************************************************************** -** Interface to code in fts5_sorter.c. -*/ -typedef struct Fts5Sorter Fts5Sorter; - -int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp); - -/* -** End of interface to code in fts5_sorter.c. -**************************************************************************/ - /************************************************************************** ** Interface to code in fts5_vocab.c. */ diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 818dfcf297..2e33d5132f 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -148,6 +148,7 @@ static void fts5HighlightAppend( */ static int fts5HighlightCb( void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ @@ -155,7 +156,10 @@ static int fts5HighlightCb( ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; - int iPos = p->iPos++; + int iPos; + + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; + iPos = p->iPos++; if( p->iRangeEnd>0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 07e1243c36..1a7c0d0f8a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -16,12 +16,14 @@ #include "fts5Int.h" int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){ - /* A no-op if an error has already occurred */ - if( *pRc ) return 1; if( (pBuf->n + nByte) > pBuf->nSpace ){ u8 *pNew; int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; + + /* A no-op if an error has already occurred */ + if( *pRc ) return 1; + while( nNew<(pBuf->n + nByte) ){ nNew = nNew * 2; } diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7e991fc21d..19e3d3ab1f 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -480,6 +480,9 @@ int sqlite3Fts5ConfigParse( pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; +#ifdef SQLITE_DEBUG + pRet->bPrefixIndex = 1; +#endif if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; @@ -645,12 +648,15 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ */ int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken); + return pConfig->pTokApi->xTokenize( + pConfig->pTok, pCtx, flags, pText, nText, xToken + ); } /* diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a713bb7c5a..559f0db82b 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -22,6 +22,8 @@ */ #define FTS5_EOF 0 +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + typedef struct Fts5ExprTerm Fts5ExprTerm; /* @@ -73,6 +75,7 @@ struct Fts5ExprTerm { int bPrefix; /* True for a prefix term */ char *zTerm; /* nul-terminated term */ Fts5IndexIter *pIter; /* Iterator for this term */ + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; /* @@ -181,6 +184,10 @@ static int fts5ExprGetToken( default: { const char *z2; + if( sqlite3Fts5IsBareword(z[0])==0 ){ + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); + return FTS5_EOF; + } tok = FTS5_STRING; for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); pToken->n = (z2 - z); @@ -244,79 +251,6 @@ int sqlite3Fts5ExprNew( return sParse.rc; } -/* -** Create a new FTS5 expression by cloning phrase iPhrase of the -** expression passed as the second argument. -*/ -int sqlite3Fts5ExprPhraseExpr( - Fts5Config *pConfig, - Fts5Expr *pExpr, - int iPhrase, - Fts5Expr **ppNew -){ - int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - Fts5ExprPhrase *pCopy; /* Copy of pOrig */ - Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pCopy = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm - ); - if( pCopy ){ - int i; /* Used to iterate through phrase terms */ - Fts5ExprPhrase **apPhrase; - Fts5ExprNode *pNode; - Fts5ExprNearset *pNear; - - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); - apPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase*) - ); - pNode = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode)); - pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) - ); - - for(i=0; inTerm; i++){ - pCopy->aTerm[i].zTerm = sqlite3Fts5Strndup(&rc, pOrig->aTerm[i].zTerm,-1); - pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - } - - if( rc==SQLITE_OK ){ - /* All the allocations succeeded. Put the expression object together. */ - pNew->pIndex = pExpr->pIndex; - pNew->pRoot = pNode; - pNew->nPhrase = 1; - pNew->apExprPhrase = apPhrase; - pNew->apExprPhrase[0] = pCopy; - - pNode->eType = (pOrig->nTerm==1 ? FTS5_TERM : FTS5_STRING); - pNode->pNear = pNear; - - pNear->nPhrase = 1; - pNear->apPhrase[0] = pCopy; - - pCopy->nTerm = pOrig->nTerm; - pCopy->pNode = pNode; - }else{ - /* At least one allocation failed. Free them all. */ - for(i=0; inTerm; i++){ - sqlite3_free(pCopy->aTerm[i].zTerm); - } - sqlite3_free(pCopy); - sqlite3_free(pNear); - sqlite3_free(pNode); - sqlite3_free(apPhrase); - sqlite3_free(pNew); - pNew = 0; - } - } - - *ppNew = pNew; - return rc; -} - /* ** Free the expression node object passed as the only argument. */ @@ -350,6 +284,114 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ return 0; } +/* +** Argument pTerm must be a synonym iterator. Return the current rowid +** that it points to. +*/ +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; + int bRetValid = 0; + Fts5ExprTerm *p; + + assert( pTerm->pSynonym ); + assert( bDesc==0 || bDesc==1 ); + for(p=pTerm; p; p=p->pSynonym){ + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( bRetValid==0 || (bDesc!=(iRowidpSynonym ); + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + const u8 *a; + int n; + i64 dummy; + rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( nIter==nAlloc ){ + int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + if( aIter!=aStatic ) sqlite3_free(aIter); + aIter = aNew; + } + sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; + } + } + + assert( *pbDel==0 ); + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + Fts5Buffer buf = {0,0,0}; + i64 iPrev = -1; + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; inTerm); /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; - int n; - const u8 *a; - rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n, &dummy); - if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){ - goto ismatch_out; + int n = 0; + int bFlag = 0; + const u8 *a = 0; + if( pTerm->pSynonym ){ + rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n); + }else{ + rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); } + if( rc!=SQLITE_OK ) goto ismatch_out; + sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]); + aIter[i].bFlag = bFlag; + if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ @@ -431,6 +481,9 @@ static int fts5ExprPhraseIsMatch( ismatch_out: *pbMatch = (pPhrase->poslist.n>0); + for(i=0; inTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } @@ -598,17 +651,55 @@ static int fts5ExprNearAdvanceFirst( int bFromValid, i64 iFrom ){ - Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; - int rc; + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; + int rc = SQLITE_OK; - assert( Fts5NodeIsString(pNode) ); - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); + + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 ii = sqlite3Fts5IterRowid(p->pIter); + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } + if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); }else{ - rc = sqlite3Fts5IterNext(pIter); + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); } - pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); return rc; } @@ -647,6 +738,35 @@ static int fts5ExprAdvanceto( return 0; } +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc /* OUT: Error code */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + int bEof = 0; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pRc = rc; + bEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); + } + return bEof; +} + /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set @@ -717,9 +837,9 @@ static int fts5ExprNearTest( ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pNear->pColset ){ + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; - rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch); + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ rc = sqlite3Fts5IterPoslistBuffer( @@ -755,6 +875,7 @@ static int fts5ExprTokenTest( assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid); @@ -801,69 +922,97 @@ static int fts5ExprNearNextMatch( i64 iLast; /* Lastest rowid any iterator points to */ int i, j; /* Phrase and token index, respectively */ int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; - assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 ); + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the ** iterator skips through rowids in the default ascending order, this means ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ - iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); + }else{ + iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); + } do { bMatch = 1; for(i=0; inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ - Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - i64 iRowid = sqlite3Fts5IterRowid(pIter); - if( iRowid!=iLast ) bMatch = 0; - if( fts5ExprAdvanceto(pIter, pExpr->bDesc, &iLast,&rc,&pNode->bEof) ){ - return rc; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + i64 iRowid = sqlite3Fts5IterRowid(pIter); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } } } } }while( bMatch==0 ); - pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); pNode->iRowid = iLast; + pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); return rc; } /* ** Initialize all term iterators in the pNear object. If any term is found -** to match no documents at all, set *pbEof to true and return immediately, -** without initializing any further iterators. +** to match no documents at all, return immediately without initializing any +** further iterators. */ static int fts5ExprNearInitAll( Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; - Fts5ExprTerm *pTerm; - Fts5ExprPhrase *pPhrase; int i, j; int rc = SQLITE_OK; for(i=0; rc==SQLITE_OK && inPhrase; i++){ - pPhrase = pNear->apPhrase[i]; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ - pTerm = &pPhrase->aTerm[j]; - if( pTerm->pIter ){ - sqlite3Fts5IterClose(pTerm->pIter); - pTerm->pIter = 0; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bEof = 1; + + for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->zTerm, strlen(p->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + &p->pIter + ); + assert( rc==SQLITE_OK || p->pIter==0 ); + if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ + bEof = 0; + } } - rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm), - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | - (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), - &pTerm->pIter - ); - assert( rc==SQLITE_OK || pTerm->pIter==0 ); - if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){ + + if( bEof ){ pNode->bEof = 1; - break; + return rc; } } } @@ -1029,10 +1178,17 @@ static int fts5ExprNodeNext( }; case FTS5_TERM: { - rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom); - if( pNode->bEof==0 ){ + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ assert( rc==SQLITE_OK ); rc = fts5ExprTokenTest(pExpr, pNode); + }else{ + pNode->bEof = 1; } return rc; }; @@ -1266,10 +1422,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ if( pPhrase ){ int i; for(i=0; inTerm; i++){ + Fts5ExprTerm *pSyn; + Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; sqlite3_free(pTerm->zTerm); - if( pTerm->pIter ){ - sqlite3Fts5IterClose(pTerm->pIter); + sqlite3Fts5IterClose(pTerm->pIter); + + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ + pNext = pSyn->pSynonym; + sqlite3Fts5IterClose(pSyn->pIter); + sqlite3_free(pSyn); } } if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); @@ -1331,6 +1493,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + int rc; }; /* @@ -1338,34 +1501,60 @@ struct TokenCtx { */ static int fts5ParseTokenize( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; - Fts5ExprTerm *pTerm; - if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ - Fts5ExprPhrase *pNew; - int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + /* If an error has already occurred, this is a no-op */ + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; - pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew - ); - if( pNew==0 ) return SQLITE_NOMEM; - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); - pCtx->pPhrase = pPhrase = pNew; - pNew->nTerm = nNew - SZALLOC; + assert( pPhrase==0 || pPhrase->nTerm>0 ); + if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ + Fts5ExprTerm *pSyn; + int nByte = sizeof(Fts5ExprTerm) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); + if( pSyn==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pSyn, 0, nByte); + pSyn->zTerm = (char*)&pSyn[1]; + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + } + }else{ + Fts5ExprTerm *pTerm; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ + Fts5ExprPhrase *pNew; + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + + pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + ); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } + } + + if( rc==SQLITE_OK ){ + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + } } - pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; - memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); - + pCtx->rc = rc; return rc; } @@ -1417,11 +1606,14 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int n; sqlite3Fts5Dequote(z); - rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize); + n = strlen(z); + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); - if( rc ){ + if( rc || (rc = sCtx.rc) ){ pParse->rc = rc; fts5ExprPhraseFree(sCtx.pPhrase); sCtx.pPhrase = 0; @@ -1450,6 +1642,79 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( return sCtx.pPhrase; } +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +int sqlite3Fts5ExprClonePhrase( + Fts5Config *pConfig, + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + int i; /* Used to iterate through phrase terms */ + + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + + TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ + + + pOrig = pExpr->apExprPhrase[iPhrase]; + + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + pNew->pRoot->eType = FTS5_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + /* ** Token pTok has appeared in a MATCH expression where the NEAR operator ** is expected. If token pTok does not contain "NEAR", store an error @@ -1630,7 +1895,10 @@ Fts5ExprNode *sqlite3Fts5ParseNode( for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } - if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ + if( pNear->nPhrase==1 + && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + ){ pRet->eType = FTS5_TERM; } }else{ @@ -1650,16 +1918,28 @@ Fts5ExprNode *sqlite3Fts5ParseNode( } static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ - char *zQuoted = sqlite3_malloc(strlen(pTerm->zTerm) * 2 + 3 + 2); + int nByte = 0; + Fts5ExprTerm *p; + char *zQuoted; + + /* Determine the maximum amount of space required. */ + for(p=pTerm; p; p=p->pSynonym){ + nByte += strlen(pTerm->zTerm) * 2 + 3 + 2; + } + zQuoted = sqlite3_malloc(nByte); + if( zQuoted ){ int i = 0; - char *zIn = pTerm->zTerm; - zQuoted[i++] = '"'; - while( *zIn ){ - if( *zIn=='"' ) zQuoted[i++] = '"'; - zQuoted[i++] = *zIn++; + for(p=pTerm; p; p=p->pSynonym){ + char *zIn = p->zTerm; + zQuoted[i++] = '"'; + while( *zIn ){ + if( *zIn=='"' ) zQuoted[i++] = '"'; + zQuoted[i++] = *zIn++; + } + zQuoted[i++] = '"'; + if( p->pSynonym ) zQuoted[i++] = '|'; } - zQuoted[i++] = '"'; if( pTerm->bPrefix ){ zQuoted[i++] = ' '; zQuoted[i++] = '*'; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a7ab7c4c05..b8aa52c2f8 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -87,7 +87,6 @@ ** + total number of segments in level. ** + for each segment from oldest to newest: ** + segment id (always > 0) -** + b-tree height (1 -> root is leaf, 2 -> root is parent of leaf etc.) ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** @@ -95,15 +94,16 @@ ** ** A single record within the %_data table. The data is a list of varints. ** The first value is the number of rows in the index. Then, for each column -** from left to right, the total number of tokens in the column for all +** from left to right, the total number of tokens in the column for all ** rows of the table. ** ** 3. Segment leaves: ** -** TERM DOCLIST FORMAT: +** TERM/DOCLIST FORMAT: ** ** Most of each segment leaf is taken up by term/doclist data. The -** general format of the term/doclist data is: +** general format of term/doclist, starting with the first term +** on the leaf page, is: ** ** varint : size of first term ** blob: first term data @@ -123,7 +123,6 @@ ** varint: rowid delta (always > 0) ** poslist: next poslist ** } -** 0x00 byte ** ** poslist format: ** @@ -143,11 +142,28 @@ ** varint: offset delta + 2 ** } ** -** PAGINATION +** PAGE FORMAT ** -** The format described above is only accurate if the entire term/doclist -** data fits on a single leaf page. If this is not the case, the format -** is changed in two ways: +** Each leaf page begins with a 4-byte header containing 2 16-bit +** unsigned integer fields in big-endian format. They are: +** +** * The byte offset of the first rowid on the page, if it exists +** and occurs before the first term (otherwise 0). +** +** * The byte offset of the start of the page footer. If the page +** footer is 0 bytes in size, then this field is the same as the +** size of the leaf page in bytes. +** +** The page footer consists of a single varint for each term located +** on the page. Each varint is the byte offset of the current term +** within the page, delta-compressed against the previous value. In +** other words, the first varint in the footer is the byte offset of +** the first term, the second is the byte offset of the second less that +** of the first, and so on. +** +** The term/doclist format described above is accurate if the entire +** term/doclist data fits on a single leaf page. If this is not the case, +** the format is changed in two ways: ** ** + if the first rowid on a page occurs before the first term, it ** is stored as a literal value: @@ -160,45 +176,6 @@ ** varint : size of first term ** blob: first term data ** -** Each leaf page begins with: -** -** + 2-byte unsigned containing offset to first rowid (or 0). -** + 2-byte unsigned containing offset to first term (or 0). -** -** Followed by term/doclist data. -** -** 4. Segment interior nodes: -** -** The interior nodes turn the list of leaves into a b+tree. -** -** Each interior node begins with a varint - the page number of the left -** most child node. Following this, for each leaf page except the first, -** the interior nodes contain: -** -** a) If the leaf page contains at least one term, then a term-prefix that -** is greater than all previous terms, and less than or equal to the -** first term on the leaf page. -** -** b) If the leaf page no terms, a record indicating how many consecutive -** leaves contain no terms, and whether or not there is an associated -** by-rowid index record. -** -** By definition, there is never more than one type (b) record in a row. -** Type (b) records only ever appear on height=1 pages - immediate parents -** of leaves. Only type (a) records are pushed to higher levels. -** -** Term format: -** -** * Number of bytes in common with previous term plus 2, as a varint. -** * Number of bytes of new term data, as a varint. -** * new term data. -** -** No-term format: -** -** * either an 0x00 or 0x01 byte. If the value 0x01 is used, then there -** is an associated index-by-rowid record. -** * the number of zero-term leaves as a varint. -** ** 5. Segment doclist indexes: ** ** Doclist indexes are themselves b-trees, however they usually consist of @@ -237,28 +214,19 @@ #define FTS5_STRUCTURE_ROWID 10 /* The structure record */ /* -** Macros determining the rowids used by segment nodes. All nodes in all -** segments for all indexes (the regular FTS index and any prefix indexes) -** are stored in the %_data table with large positive rowids. +** Macros determining the rowids used by segment leaves and dlidx leaves +** and nodes. All nodes and leaves are stored in the %_data table with large +** positive rowids. ** -** The %_data table may contain up to (1<szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ +) + #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 +/* +** Argument is a pointer to an Fts5Data structure that contains a leaf +** page. This macro evaluates to true if the leaf contains no terms, or +** false if it contains at least one term. +*/ +#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) + +#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) + +#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) + /* ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. @@ -526,24 +521,6 @@ struct Fts5IndexIter { }; -/* -** Object for iterating through the conents of a single internal node in -** memory. -*/ -struct Fts5NodeIter { - /* Internal. Set and managed by fts5NodeIterXXX() functions. Except, - ** the EOF test for the iterator is (Fts5NodeIter.aData==0). */ - const u8 *aData; - int nData; - int iOff; - - /* Output variables */ - Fts5Buffer term; - int nEmpty; - int iChild; - int bDlidx; -}; - /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -573,23 +550,6 @@ struct Fts5DlidxIter { Fts5DlidxLvl aLvl[1]; }; - - -/* -** The first argument passed to this macro is a pointer to an Fts5Buffer -** object. -*/ -#define fts5BufferSize(pBuf,n) { \ - if( pBuf->nSpacep, n); \ - if( pNew==0 ){ \ - sqlite3_free(pBuf->p); \ - } \ - pBuf->nSpace = n; \ - pBuf->p = pNew; \ - } \ -} - static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -617,6 +577,7 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){ ** ** res = *pLeft - *pRight */ +#ifdef SQLITE_DEBUG static int fts5BufferCompareBlob( Fts5Buffer *pLeft, /* Left hand side of comparison */ const u8 *pRight, int nRight /* Right hand side of comparison */ @@ -625,7 +586,7 @@ static int fts5BufferCompareBlob( int res = memcmp(pLeft->p, pRight, nCmp); return (res==0 ? (pLeft->n - nRight) : res); } - +#endif /* ** Compare the contents of the two buffers using memcmp(). If one buffer @@ -653,6 +614,11 @@ static int fts5BlobCompare( } #endif +static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ + int ret; + fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); + return ret; +} /* ** Close the read-only blob handle, if it is open. @@ -665,11 +631,14 @@ static void fts5CloseReader(Fts5Index *p){ } } -static Fts5Data *fts5DataReadOrBuffer( - Fts5Index *p, - Fts5Buffer *pBuf, - i64 iRowid -){ + +/* +** Retrieve a record from the %_data table. +** +** If an error occurs, NULL is returned and an error left in the +** Fts5Index object. +*/ +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = 0; if( p->rc==SQLITE_OK ){ int rc = SQLITE_OK; @@ -689,8 +658,8 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } - /* If the blob handle is not yet open, open and seek it. Otherwise, use - ** the blob_reopen() API to reseek the existing blob handle. */ + /* If the blob handle is not open at this point, open it and seek + ** to the requested entry. */ if( p->pReader==0 && rc==SQLITE_OK ){ Fts5Config *pConfig = p->pConfig; rc = sqlite3_blob_open(pConfig->db, @@ -708,22 +677,13 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ int nByte = sqlite3_blob_bytes(p->pReader); - if( pBuf ){ - fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20); - pBuf->n = nByte; - aOut = pBuf->p; - if( aOut==0 ){ - rc = SQLITE_NOMEM; - } + int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc(nAlloc); + if( pRet ){ + pRet->nn = nByte; + aOut = pRet->p = (u8*)&pRet[1]; }else{ - int nSpace = nByte + FTS5_DATA_PADDING; - pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data)); - if( pRet ){ - pRet->n = nByte; - aOut = pRet->p = (u8*)&pRet[1]; - }else{ - rc = SQLITE_NOMEM; - } + rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ @@ -732,39 +692,19 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc!=SQLITE_OK ){ sqlite3_free(pRet); pRet = 0; + }else{ + /* TODO1: Fix this */ + pRet->szLeaf = fts5GetU16(&pRet->p[2]); } } p->rc = rc; p->nRead++; } - return pRet; -} - -/* -** Retrieve a record from the %_data table. -** -** If an error occurs, NULL is returned and an error left in the -** Fts5Index object. -*/ -static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ - Fts5Data *pRet = fts5DataReadOrBuffer(p, 0, iRowid); assert( (pRet==0)==(p->rc!=SQLITE_OK) ); return pRet; } -/* -** Read a record from the %_data table into the buffer supplied as the -** second argument. -** -** If an error occurs, an error is left in the Fts5Index object. If an -** error has already occurred when this function is called, it is a -** no-op. -*/ -static void fts5DataBuffer(Fts5Index *p, Fts5Buffer *pBuf, i64 iRowid){ - (void)fts5DataReadOrBuffer(p, pBuf, iRowid); -} - /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). @@ -797,7 +737,6 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ - int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", @@ -849,8 +788,8 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ ** Remove all records associated with segment iSegid. */ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ - i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0); - i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1; + i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); + i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; fts5DataDelete(p, iFirst, iLast); if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; @@ -947,7 +886,6 @@ static int fts5StructureDecode( pLvl->nSeg = nTotal; for(iSeg=0; iSegaSeg[iSeg].iSegid); - i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); } @@ -1033,19 +971,18 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p){ Fts5Config *pConfig = p->pConfig; Fts5Structure *pRet = 0; /* Object to return */ int iCookie; /* Configuration cookie */ - Fts5Buffer buf = {0, 0, 0}; - - fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID); - if( buf.p==0 ) return 0; - assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) ); - memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING); - p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet); + Fts5Data *pData; + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc ) return 0; + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } - fts5BufferFree(&buf); + fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; @@ -1104,7 +1041,6 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ for(iSeg=0; iSegnSeg; iSeg++){ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nHeight); fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); } @@ -1193,8 +1129,9 @@ static void fts5StructurePromote( int szPromote = 0; /* Promote anything this size or smaller */ Fts5StructureSegment *pSeg; /* Segment just written */ int szSeg; /* Size of segment just written */ + int nSeg = pStruct->aLevel[iLvl].nSeg; - + if( nSeg==0 ) return; pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); @@ -1228,62 +1165,6 @@ static void fts5StructurePromote( } -/* -** If the pIter->iOff offset currently points to an entry indicating one -** or more term-less nodes, advance past it and set pIter->nEmpty to -** the number of empty child nodes. -*/ -static void fts5NodeIterGobbleNEmpty(Fts5NodeIter *pIter){ - if( pIter->iOffnData && 0==(pIter->aData[pIter->iOff] & 0xfe) ){ - pIter->bDlidx = pIter->aData[pIter->iOff] & 0x01; - pIter->iOff++; - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty); - }else{ - pIter->nEmpty = 0; - pIter->bDlidx = 0; - } -} - -/* -** Advance to the next entry within the node. -*/ -static void fts5NodeIterNext(int *pRc, Fts5NodeIter *pIter){ - if( pIter->iOff>=pIter->nData ){ - pIter->aData = 0; - pIter->iChild += pIter->nEmpty; - }else{ - int nPre, nNew; - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nPre); - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nNew); - pIter->term.n = nPre-2; - fts5BufferAppendBlob(pRc, &pIter->term, nNew, pIter->aData+pIter->iOff); - pIter->iOff += nNew; - pIter->iChild += (1 + pIter->nEmpty); - fts5NodeIterGobbleNEmpty(pIter); - if( *pRc ) pIter->aData = 0; - } -} - - -/* -** Initialize the iterator object pIter to iterate through the internal -** segment node in pData. -*/ -static void fts5NodeIterInit(const u8 *aData, int nData, Fts5NodeIter *pIter){ - memset(pIter, 0, sizeof(*pIter)); - pIter->aData = aData; - pIter->nData = nData; - pIter->iOff = fts5GetVarint32(aData, pIter->iChild); - fts5NodeIterGobbleNEmpty(pIter); -} - -/* -** Free any memory allocated by the iterator object. -*/ -static void fts5NodeIterFree(Fts5NodeIter *pIter){ - fts5BufferFree(&pIter->term); -} - /* ** Advance the iterator passed as the only argument. If the end of the ** doclist-index page is reached, return non-zero. @@ -1299,11 +1180,11 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ pLvl->iFirstOff = pLvl->iOff; }else{ int iOff; - for(iOff=pLvl->iOff; iOffn; iOff++){ + for(iOff=pLvl->iOff; iOffnn; iOff++){ if( pData->p[iOff] ) break; } - if( iOffn ){ + if( iOffnn ){ i64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); @@ -1534,11 +1415,6 @@ static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){ return pIter->aLvl[0].iLeafPgno; } -static void fts5LeafHeader(Fts5Data *pLeaf, int *piRowid, int *piTerm){ - *piRowid = (int)fts5GetU16(&pLeaf->p[0]); - *piTerm = (int)fts5GetU16(&pLeaf->p[2]); -} - /* ** Load the next leaf page into the segment iterator. */ @@ -1546,20 +1422,32 @@ static void fts5SegIterNextPage( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter /* Iterator to advance to next page */ ){ + Fts5Data *pLeaf; Fts5StructureSegment *pSeg = pIter->pSeg; fts5DataRelease(pIter->pLeaf); pIter->iLeafPgno++; if( pIter->pNextLeaf ){ - assert( pIter->iLeafPgno<=pSeg->pgnoLast ); pIter->pLeaf = pIter->pNextLeaf; pIter->pNextLeaf = 0; }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ pIter->pLeaf = fts5DataRead(p, - FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pIter->iLeafPgno) + FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) ); }else{ pIter->pLeaf = 0; } + pLeaf = pIter->pLeaf; + + if( pLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf; + if( fts5LeafIsTermless(pLeaf) ){ + pIter->iEndofDoclist = pLeaf->nn+1; + }else{ + pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff], + pIter->iEndofDoclist + ); + } + } } /* @@ -1591,7 +1479,8 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ - if( iOff>=pIter->pLeaf->n ){ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ const u8 *a = &pIter->pLeaf->p[iOff]; @@ -1604,7 +1493,8 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; - if( iOff>=pIter->pLeaf->n ){ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -1645,6 +1535,14 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ pIter->iTermLeafPgno = pIter->iLeafPgno; pIter->iLeafOffset = iOff; + if( pIter->iPgidxOff>=pIter->pLeaf->nn ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); + pIter->iEndofDoclist += nExtra; + } + fts5SegIterLoadRowid(p, pIter); } @@ -1679,8 +1577,10 @@ static void fts5SegIterInit( } if( p->rc==SQLITE_OK ){ - u8 *a = pIter->pLeaf->p; - pIter->iLeafOffset = fts5GetU16(&a[2]); + pIter->iLeafOffset = 4; + assert_nc( pIter->pLeaf->nn>4 ); + assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); + pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); } @@ -1702,11 +1602,16 @@ static void fts5SegIterInit( ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ - int n = pIter->pLeaf->n; + int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; + if( n>pIter->iEndofDoclist ){ + n = pIter->iEndofDoclist; + } + + ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; int nPos; @@ -1716,7 +1621,6 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ i += nPos; if( i>=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); - if( iDelta==0 ) break; pIter->iRowid += iDelta; if( iRowidOffset>=pIter->nRowidOffset ){ @@ -1750,17 +1654,20 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pNew; pIter->iLeafPgno--; pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( - pIter->pSeg->iSegid, 0, pIter->iLeafPgno + pIter->pSeg->iSegid, pIter->iLeafPgno )); if( pNew ){ - if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ - if( pIter->iTermLeafOffsetn ){ - pIter->pLeaf = pNew; - pIter->iLeafOffset = pIter->iTermLeafOffset; - } + /* iTermLeafOffset may be equal to szLeaf if the term is the last + ** thing on the page - i.e. the first rowid is on the following page. + ** In this case leaf pIter->pLeaf==0, this iterator is at EOF. */ + if( pIter->iLeafPgno==pIter->iTermLeafPgno + && pIter->iTermLeafOffsetszLeaf + ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; }else{ - int iRowidOff, dummy; - fts5LeafHeader(pNew, &iRowidOff, &dummy); + int iRowidOff; + iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -1778,6 +1685,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ } if( pIter->pLeaf ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; fts5SegIterReverseInitPage(p, pIter); } } @@ -1833,26 +1741,27 @@ static void fts5SegIterNext( /* Search for the end of the position list within the current page. */ u8 *a = pLeaf->p; - int n = pLeaf->n; + int n = pLeaf->szLeaf; + ASSERT_SZLEAF_OK(pLeaf); iOff = pIter->iLeafOffset + pIter->nPos; if( iOffiLeafOffset = iOff; - if( iDelta==0 ){ + /* The next entry is on the current page. */ + assert_nc( iOff<=pIter->iEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ bNewTerm = 1; - if( iOff>=n ){ - fts5SegIterNextPage(p, pIter); - pIter->iLeafOffset = 4; - }else if( iOff!=fts5GetU16(&a[2]) ){ - pIter->iLeafOffset += fts5GetVarint32(&a[iOff], nKeep); + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); } }else{ + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); } + pIter->iLeafOffset = iOff; + }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; @@ -1866,7 +1775,9 @@ static void fts5SegIterNext( pIter->pLeaf = 0; }else{ pIter->pLeaf->p = (u8*)pList; - pIter->pLeaf->n = nList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -1877,15 +1788,27 @@ static void fts5SegIterNext( fts5SegIterNextPage(p, pIter); pLeaf = pIter->pLeaf; if( pLeaf==0 ) break; - if( (iOff = fts5GetU16(&pLeaf->p[0])) && iOffn ){ + ASSERT_SZLEAF_OK(pLeaf); + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; + + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + ); + } + } - else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){ + else if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff + ); pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; bNewTerm = 1; } - if( iOff>=pLeaf->n ){ + if( iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } @@ -1926,57 +1849,37 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ if( pDlidx ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); - pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast)); + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ - int iOff; /* Byte offset within pLeaf */ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ - /* Currently, Fts5SegIter.iLeafOffset (and iOff) points to the first - ** byte of position-list content for the current rowid. Back it up - ** so that it points to the start of the position-list size field. */ + /* Currently, Fts5SegIter.iLeafOffset points to the first byte of + ** position-list content for the current rowid. Back it up so that it + ** points to the start of the position-list size field. */ pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); - iOff = pIter->iLeafOffset; - assert( iOff>=4 ); - - /* Search for a new term within the current leaf. If one can be found, - ** then this page contains the largest rowid for the current term. */ - while( iOffn ){ - int nPos; - i64 iDelta; - int bDummy; - - /* Read the position-list size field */ - iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); - iOff += nPos; - if( iOff>=pLeaf->n ) break; - - /* Rowid delta. Or, if 0x00, the end of doclist marker. */ - nPos = fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta); - if( iDelta==0 ) break; - iOff += nPos; - } /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ - if( iOff>=pLeaf->n ){ + if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ int pgno; Fts5StructureSegment *pSeg = pIter->pSeg; /* The last rowid in the doclist may not be on the current page. Search ** forward to find the page containing the last rowid. */ for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ - i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pgno); + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); Fts5Data *pNew = fts5DataRead(p, iAbs); if( pNew ){ - int iRowid, iTerm; - fts5LeafHeader(pNew, &iRowid, &iTerm); + int iRowid, bTermless; + iRowid = fts5LeafFirstRowidOff(pNew); + bTermless = fts5LeafIsTermless(pNew); if( iRowid ){ SWAPVAL(Fts5Data*, pNew, pLast); pgnoLast = pgno; } fts5DataRelease(pNew); - if( iTerm ) break; + if( bTermless==0 ) break; } } } @@ -1992,14 +1895,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ** first rowid on this page. */ if( pLast ){ - int dummy; int iOff; fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - fts5LeafHeader(pLast, &iOff, &dummy); + iOff = fts5LeafFirstRowidOff(pLast); iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; + + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } + } fts5SegIterReverseInitPage(p, pIter); @@ -2022,143 +1931,20 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ /* Check if the current doclist ends on this page. If it does, return ** early without loading the doclist-index (as it belongs to a different ** term. */ - if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ - int iOff = pIter->iLeafOffset + pIter->nPos; - while( iOffn ){ - int bDummy; - int nPos; - i64 iDelta; - - /* iOff is currently the offset of the start of position list data */ - iOff += fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta); - if( iDelta==0 ) return; - assert_nc( iOffn ); - iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); - iOff += nPos; - } + if( pIter->iTermLeafPgno==pIter->iLeafPgno + && pIter->iEndofDoclistszLeaf + ){ + return; } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#ifdef SQLITE_DEBUG -static void fts5AssertNodeSeekOk( - Fts5Buffer *pNode, - const u8 *pTerm, int nTerm, /* Term to search for */ - int iExpectPg, - int bExpectDlidx -){ - int bDlidx; - int iPg; - int rc = SQLITE_OK; - Fts5NodeIter node; - - fts5NodeIterInit(pNode->p, pNode->n, &node); - assert( node.term.n==0 ); - iPg = node.iChild; - bDlidx = node.bDlidx; - for(fts5NodeIterNext(&rc, &node); - node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0; - fts5NodeIterNext(&rc, &node) - ){ - iPg = node.iChild; - bDlidx = node.bDlidx; - } - fts5NodeIterFree(&node); - - assert( rc!=SQLITE_OK || iPg==iExpectPg ); - assert( rc!=SQLITE_OK || bDlidx==bExpectDlidx ); -} -#else -#define fts5AssertNodeSeekOk(v,w,x,y,z) -#endif - -/* -** Argument pNode is an internal b-tree node. This function searches -** within the node for the largest term that is smaller than or equal -** to (pTerm/nTerm). -** -** It returns the associated page number. Or, if (pTerm/nTerm) is smaller -** than all terms within the node, the leftmost child page number. -** -** Before returning, (*pbDlidx) is set to true if the last term on the -** returned child page number has a doclist-index. Or left as is otherwise. -*/ -static int fts5NodeSeek( - Fts5Buffer *pNode, /* Node to search */ - const u8 *pTerm, int nTerm, /* Term to search for */ - int *pbDlidx /* OUT: True if dlidx flag is set */ -){ - int iPg; - u8 *pPtr = pNode->p; - u8 *pEnd = &pPtr[pNode->n]; - int nMatch = 0; /* Number of bytes of pTerm already matched */ - - assert( *pbDlidx==0 ); - - pPtr += fts5GetVarint32(pPtr, iPg); - while( pPtr=pEnd ) break; - } - - /* Read the next "term" pointer. Set nKeep to the number of bytes to - ** keep from the previous term, and nNew to the number of bytes of - ** new data that will be appended to it. */ - nKeep = (int)*pPtr++; - nNew = (int)*pPtr++; - if( (nKeep | nNew) & 0x0080 ){ - pPtr -= 2; - pPtr += fts5GetVarint32(pPtr, nKeep); - pPtr += fts5GetVarint32(pPtr, nNew); - } - nKeep -= 2; - - /* Compare (pTerm/nTerm) to the current term on the node (the one described - ** by nKeep/nNew). If the node term is larger, break out of the while() - ** loop. - ** - ** Otherwise, if (pTerm/nTerm) is larger or the two terms are equal, - ** leave variable nMatch set to the size of the largest prefix common to - ** both terms in bytes. */ - if( nKeep==nMatch ){ - int nTst = MIN(nNew, nTerm-nMatch); - int i; - for(i=0; i pTerm[nMatch]) ) break; - }else if( nKeeppLeaf->p; - int n = pIter->pLeaf->n; + int szLeaf = pIter->pLeaf->szLeaf; + int n = pIter->pLeaf->nn; int nMatch = 0; int nKeep = 0; int nNew = 0; + int iTermOff; + int iPgidx; /* Current offset in pgidx */ + int bEndOfPage = 0; assert( p->rc==SQLITE_OK ); - assert( pIter->pLeaf ); - iOff = fts5GetU16(&a[2]); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - return; - } + iPgidx = szLeaf; + iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); + iOff = iTermOff; while( 1 ){ - int i; - int nCmp; /* Figure out how many new bytes are in this term */ fts5IndexGetVarint32(a, iOff, nNew); - if( nKeep=nMatch ); if( nKeep==nMatch ){ + int nCmp; + int i; nCmp = MIN(nNew, nTerm-nMatch); for(i=0; i=n ){ + bEndOfPage = 1; + break; + } - /* Skip past rowid delta */ - fts5IndexSkipVarint(a, iOff); - - /* Skip past position list */ - fts5IndexGetVarint32(a, iOff, nPos); - iOff += (nPos >> 1); - if( iOff>=(n-1) ){ - iOff = n; - goto search_failed; - } - - /* If this is the end of the doclist, break out of the loop */ - if( a[iOff]==0x00 ){ - iOff++; - break; - } - }; + iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); + iTermOff += nKeep; + iOff = iTermOff; /* Read the nKeep field of the next term. */ fts5IndexGetVarint32(a, iOff, nKeep); @@ -2266,14 +2038,14 @@ static void fts5LeafSeek( fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; return; - }else if( iOff>=n ){ + }else if( bEndOfPage ){ do { fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; - iOff = fts5GetU16(&a[2]); - if( iOff ){ - if( iOff<4 || iOff>=n ){ + if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ + fts5GetVarint32(&pIter->pLeaf->p[pIter->pLeaf->szLeaf], iOff); + if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ nKeep = 0; @@ -2285,6 +2057,7 @@ static void fts5LeafSeek( } search_success: + pIter->iLeafOffset = iOff + nNew; pIter->iTermLeafOffset = pIter->iLeafOffset; pIter->iTermLeafPgno = pIter->iLeafPgno; @@ -2292,6 +2065,15 @@ static void fts5LeafSeek( fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + if( iPgidx>=n ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + iPgidx += fts5GetVarint32(&a[iPgidx], nExtra); + pIter->iEndofDoclist = iTermOff + nExtra; + } + pIter->iPgidxOff = iPgidx; + fts5SegIterLoadRowid(p, pIter); fts5SegIterLoadNPos(p, pIter); } @@ -2424,9 +2206,10 @@ static void fts5SegIterHashInit( pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf==0 ) return; pLeaf->p = (u8*)pList; - pLeaf->n = nList; + pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pLeaf->nn+1; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; @@ -2617,9 +2400,9 @@ static void fts5SegIterGotoPage( if( p->rc==SQLITE_OK ){ int iOff; u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->n; + int n = pIter->pLeaf->szLeaf; - iOff = fts5GetU16(&a[0]); + iOff = fts5LeafFirstRowidOff(pIter->pLeaf); if( iOff<4 || iOff>=n ){ p->rc = FTS5_CORRUPT; }else{ @@ -2677,13 +2460,13 @@ static void fts5SegIterNextFrom( } } - while( p->rc==SQLITE_OK ){ + do{ if( bMove ) fts5SegIterNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; bMove = 1; - } + }while( p->rc==SQLITE_OK ); } @@ -2951,9 +2734,10 @@ static void fts5MultiIterNew2( Fts5SegIter *pIter = &pNew->aSeg[1]; pIter->flags = FTS5_SEGITER_ONETERM; - if( pData->n>0 ){ + if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; pNew->aFirst[1].iFirst = 1; if( bDesc ){ pNew->bRev = 1; @@ -3031,7 +2815,7 @@ static void fts5ChunkIterate( int nRem = pSeg->nPos; /* Number of bytes still to come */ Fts5Data *pData = 0; u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - int nChunk = MIN(nRem, pSeg->pLeaf->n - pSeg->iLeafOffset); + int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); int pgno = pSeg->iLeafPgno; int pgnoSave = 0; @@ -3047,10 +2831,10 @@ static void fts5ChunkIterate( break; }else{ pgno++; - pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, 0, pgno)); + pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); if( pData==0 ) break; pChunk = &pData->p[4]; - nChunk = MIN(nRem, pData->n - 4); + nChunk = MIN(nRem, pData->szLeaf - 4); if( pgno==pgnoSave ){ assert( pSeg->pNextLeaf==0 ); pSeg->pNextLeaf = pData; @@ -3336,19 +3120,30 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); + + /* Set the szLeaf header field. */ + assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + fts5PutU16(&pPage->buf.p[2], pPage->buf.n); + if( pWriter->bFirstTermInPage ){ /* No term was written to this page. */ - assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + assert( pPage->pgidx.n==0 ); fts5WriteBtreeNoTerm(p, pWriter); + }else{ + /* Append the pgidx to the page buffer. Set the szLeaf header field. */ + fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p); } - /* Write the current page to the db. */ - iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, 0, pPage->pgno); + /* Write the page out to disk */ + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno); fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); /* Initialize the next page. */ fts5BufferZero(&pPage->buf); + fts5BufferZero(&pPage->pgidx); fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); + pPage->iPrevPgidx = 0; pPage->pgno++; /* Increase the leaves written counter */ @@ -3373,20 +3168,31 @@ static void fts5WriteAppendTerm( ){ int nPrefix; /* Bytes of prefix compression for term */ Fts5PageWriter *pPage = &pWriter->writer; + Fts5Buffer *pPgidx = &pWriter->writer.pgidx; - assert( pPage->buf.n==0 || pPage->buf.n>4 ); - if( pPage->buf.n==0 ){ - /* Zero the first term and first rowid fields */ - static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; - fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); - assert( pWriter->bFirstTermInPage ); + assert( p->rc==SQLITE_OK ); + assert( pPage->buf.n>=4 ); + assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); + + /* If the current leaf page is full, flush it to disk. */ + if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ + if( pPage->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + } + fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); } - if( p->rc ) return; + /* TODO1: Updating pgidx here. */ + pPgidx->n += sqlite3Fts5PutVarint( + &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx + ); + pPage->iPrevPgidx = pPage->buf.n; +#if 0 + fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); + pPgidx->n += 2; +#endif + if( pWriter->bFirstTermInPage ){ - /* Update the "first term" field of the page header. */ - assert( pPage->buf.p[2]==0 && pPage->buf.p[3]==0 ); - fts5PutU16(&pPage->buf.p[2], pPage->buf.n); nPrefix = 0; if( pPage->pgno!=1 ){ /* This is the first term on a leaf that is not the leftmost leaf in @@ -3428,11 +3234,6 @@ static void fts5WriteAppendTerm( assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) ); pWriter->aDlidx[0].pgno = pPage->pgno; - - /* If the current leaf page is full, flush it to disk. */ - if( pPage->buf.n>=p->pConfig->pgsz ){ - fts5WriteFlushLeaf(p, pWriter); - } } /* @@ -3447,6 +3248,10 @@ static void fts5WriteAppendRowid( if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ + fts5WriteFlushLeaf(p, pWriter); + } + /* If this is to be the first rowid written to the page, set the ** rowid-pointer in the page-header. Also append a value to the dlidx ** buffer, in case a doclist-index is required. */ @@ -3467,10 +3272,6 @@ static void fts5WriteAppendRowid( pWriter->bFirstRowidInPage = 0; fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); - - if( pPage->buf.n>=p->pConfig->pgsz ){ - fts5WriteFlushLeaf(p, pWriter); - } } } @@ -3485,8 +3286,10 @@ static void fts5WriteAppendPoslistData( int n = nData; assert( p->pConfig->pgsz>0 ); - while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){ - int nReq = p->pConfig->pgsz - pPage->buf.n; + while( p->rc==SQLITE_OK + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz + ){ + int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; int nCopy = 0; while( nCopyrc, &pWriter->writer.buf, 0); -} - /* ** Flush any data cached by the writer object to the database. Free any ** allocations associated with the writer. @@ -3513,27 +3312,21 @@ static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ static void fts5WriteFinish( Fts5Index *p, Fts5SegWriter *pWriter, /* Writer object */ - int *pnHeight, /* OUT: Height of the b-tree */ int *pnLeaf /* OUT: Number of leaf pages in b-tree */ ){ int i; Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ - if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){ - *pnLeaf = 0; - *pnHeight = 0; - }else{ - if( pLeaf->buf.n>4 ){ - fts5WriteFlushLeaf(p, pWriter); - } - *pnLeaf = pLeaf->pgno-1; - - fts5WriteFlushBtree(p, pWriter); - *pnHeight = 0; + assert( pLeaf->pgno>=1 ); + if( pLeaf->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); } + *pnLeaf = pLeaf->pgno-1; + fts5WriteFlushBtree(p, pWriter); } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pLeaf->pgidx); fts5BufferFree(&pWriter->btterm); for(i=0; inDlidx; i++){ @@ -3547,6 +3340,8 @@ static void fts5WriteInit( Fts5SegWriter *pWriter, int iSegid ){ + const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; + memset(pWriter, 0, sizeof(Fts5SegWriter)); pWriter->iSegid = iSegid; @@ -3555,6 +3350,10 @@ static void fts5WriteInit( pWriter->bFirstTermInPage = 1; pWriter->iBtPage = 1; + /* Grow the two buffers to pgsz + padding bytes in size. */ + fts5BufferGrow(&p->rc, &pWriter->writer.pgidx, nBuffer); + fts5BufferGrow(&p->rc, &pWriter->writer.buf, nBuffer); + if( p->pIdxWriter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( @@ -3564,6 +3363,13 @@ static void fts5WriteInit( } if( p->rc==SQLITE_OK ){ + /* Initialize the 4-byte leaf-page header to 0x00. */ + memset(pWriter->writer.buf.p, 0, 4); + pWriter->writer.buf.n = 4; + + /* Bind the current output segment id to the index-writer. This is an + ** optimization over binding the same value over and over as rows are + ** inserted into %_idx by the current writer. */ sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); } } @@ -3592,19 +3398,37 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ i64 iLeafRowid; Fts5Data *pData; int iId = pSeg->pSeg->iSegid; - u8 aHdr[4] = {0x00, 0x00, 0x00, 0x04}; + u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; - iLeafRowid = FTS5_SEGMENT_ROWID(iId, 0, pSeg->iTermLeafPgno); + iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); pData = fts5DataRead(p, iLeafRowid); if( pData ){ fts5BufferZero(&buf); + fts5BufferGrow(&p->rc, &buf, pData->nn); fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff, &pData->p[iOff]); + if( p->rc==SQLITE_OK ){ + /* Set the szLeaf field */ + fts5PutU16(&buf.p[2], buf.n); + } + + /* Set up the new page-index array */ + fts5BufferAppendVarint(&p->rc, &buf, 4); + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno + && pSeg->iEndofDoclistszLeaf + ){ + int nDiff = pData->szLeaf - pSeg->iEndofDoclist; + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); + fts5BufferAppendBlob(&p->rc, &buf, + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] + ); + } + fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; - fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid); + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } @@ -3639,7 +3463,6 @@ static void fts5IndexMergeLevel( Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; - int bRequireDoclistTerm = 0; /* Doclist terminator (0x00) required */ int bOldest; /* True if the output segment is the oldest */ assert( iLvlnLevel ); @@ -3704,12 +3527,8 @@ static void fts5IndexMergeLevel( } /* This is a new term. Append a term to the output segment. */ - if( bRequireDoclistTerm ){ - fts5WriteAppendZerobyte(p, &writer); - } fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); - bRequireDoclistTerm = 1; } /* Append the rowid to the output */ @@ -3723,7 +3542,7 @@ static void fts5IndexMergeLevel( /* Flush the last leaf page to disk. Set the output segment b-tree height ** and last leaf page number at the same time. */ - fts5WriteFinish(p, &writer, &pSeg->nHeight, &pSeg->pgnoLast); + fts5WriteFinish(p, &writer, &pSeg->pgnoLast); if( fts5MultiIterEof(p, pIter) ){ int i; @@ -3848,6 +3667,7 @@ static void fts5IndexCrisismerge( assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); fts5StructurePromote(p, iLvl+1, pStruct); iLvl++; } @@ -3875,10 +3695,12 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ int ret; u32 dummy; ret = fts5GetVarint32(aBuf, dummy); - while( 1 ){ - int i = fts5GetVarint32(&aBuf[ret], dummy); - if( (ret + i) > nMax ) break; - ret += i; + if( ret nMax ) break; + ret += i; + } } return ret; } @@ -3911,75 +3733,36 @@ static void fts5FlushOneHash(Fts5Index *p){ const int pgsz = p->pConfig->pgsz; Fts5StructureSegment *pSeg; /* New segment within pStruct */ - int nHeight; /* Height of new segment b-tree */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ - const u8 *zPrev = 0; + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); - /* Pre-allocate the buffer used to assemble leaf pages to the target - ** page size. */ - assert( pgsz>0 ); pBuf = &writer.writer.buf; - fts5BufferGrow(&p->rc, pBuf, pgsz + 20); + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); /* Begin scanning through hash table entries. This loop runs once for each ** term/doclist currently stored within the hash table. */ if( p->rc==SQLITE_OK ){ - memset(pBuf->p, 0, 4); - pBuf->n = 4; p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ - int nTerm; /* Size of zTerm in bytes */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ - int nSuffix; /* Size of term suffix */ + /* Write the term for this entry to disk. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - nTerm = strlen(zTerm); + fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm); - /* Decide if the term will fit on the current leaf. If it will not, - ** flush the leaf to disk here. */ - if( pBuf->n>4 && (pBuf->n + nTerm + 2) > pgsz ){ - fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.writer.buf; - if( (nTerm + 32) > pBuf->nSpace ){ - fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); - if( p->rc ) break; - } - } - - /* Write the term to the leaf. And if it is the first on the leaf, and - ** the leaf is not page number 1, push it up into the b-tree hierarchy - ** as well. */ - if( writer.bFirstTermInPage==0 ){ - int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nPre); - nSuffix = nTerm - nPre; - }else{ - fts5PutU16(&pBuf->p[2], pBuf->n); - writer.bFirstTermInPage = 0; - if( writer.writer.pgno!=1 ){ - int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); - fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm); - pBuf = &writer.writer.buf; - assert( nPren += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nSuffix); - fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix); - - /* We just wrote a term into page writer.aWriter[0].pgno. If a - ** doclist-index is to be generated for this doclist, it will be - ** associated with this page. */ - assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 ); - writer.aDlidx[0].pgno = writer.writer.pgno; - - if( pgsz>=(pBuf->n + nDoclist + 1) ){ + assert( writer.bFirstRowidInPage==0 ); + if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ @@ -3987,8 +3770,6 @@ static void fts5FlushOneHash(Fts5Index *p){ i64 iDelta = 0; int iOff = 0; - writer.bFirstRowidInPage = 0; - /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ @@ -4011,7 +3792,7 @@ static void fts5FlushOneHash(Fts5Index *p){ } assert( pBuf->n<=pBuf->nSpace ); - if( (pBuf->n + nCopy) <= pgsz ){ + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy ** it in one go. */ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); @@ -4022,7 +3803,7 @@ static void fts5FlushOneHash(Fts5Index *p){ const u8 *pPoslist = &pDoclist[iOff]; int iPos = 0; while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n; + int nSpace = pgsz - pBuf->n - pPgidx->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ n = nCopy - iPos; @@ -4032,9 +3813,8 @@ static void fts5FlushOneHash(Fts5Index *p){ assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; - if( pBuf->n>=pgsz ){ + if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.writer.buf; } if( iPos>=nCopy ) break; } @@ -4043,13 +3823,13 @@ static void fts5FlushOneHash(Fts5Index *p){ } } - pBuf->p[pBuf->n++] = '\0'; + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); - zPrev = (const u8*)zTerm; sqlite3Fts5HashScanNext(pHash); } sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &nHeight, &pgnoLast); + fts5WriteFinish(p, &writer, &pgnoLast); /* Update the Fts5Structure. It is written back to the database by the ** fts5StructureRelease() call below. */ @@ -4060,7 +3840,6 @@ static void fts5FlushOneHash(Fts5Index *p){ if( p->rc==SQLITE_OK ){ pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; pSeg->iSegid = iSegid; - pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; pStruct->nSegment++; @@ -4162,7 +3941,10 @@ static void fts5PoslistCallback( void *pCtx, const u8 *pChunk, int nChunk ){ - fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk); + } } /* @@ -4368,7 +4150,7 @@ static void fts5SetupPrefixIter( i64 iRowid = fts5MultiIterRowid(p1); int nTerm; const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); - assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( nTerm0 && iRowid<=iLastRowid ){ @@ -4397,7 +4179,7 @@ static void fts5SetupPrefixIter( pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ pData->p = (u8*)&pData[1]; - pData->n = doclist.n; + pData->nn = pData->szLeaf = doclist.n; memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } @@ -4459,13 +4241,9 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; - - assert( p->rc==SQLITE_OK ); - p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); - memset(&s, 0, sizeof(Fts5Structure)); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); - return fts5IndexReturn(p); } @@ -4631,7 +4409,12 @@ int sqlite3Fts5IndexQuery( memcpy(&buf.p[1], pToken, nToken); #ifdef SQLITE_DEBUG - if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ + /* If the QUERY_TEST_NOIDX flag was specified, then this must be a + ** prefix-query. Instead of using a prefix-index (if one exists), + ** evaluate the prefix query using the main FTS index. This is used + ** for internal sanity checking by the integrity-check in debug + ** mode only. */ + if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; }else @@ -4751,7 +4534,7 @@ int sqlite3Fts5IterPoslist( assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; *pn = pSeg->nPos; - if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->n ){ + if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){ *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset]; }else{ fts5BufferZero(&pIter->poslist); @@ -4787,13 +4570,28 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ } /* -** Read the "averages" record into the buffer supplied as the second -** argument. Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. +** Read and decode the "averages" record from the database. +** +** Parameter anSize must point to an array of size nCol, where nCol is +** the number of user defined columns in the FTS table. */ -int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){ - assert( p->rc==SQLITE_OK ); - fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ + int nCol = p->pConfig->nCol; + Fts5Data *pData; + + *pnRow = 0; + memset(anSize, 0, sizeof(i64) * nCol); + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); + if( p->rc==SQLITE_OK && pData->nn ){ + int i = 0; + int iCol; + i += fts5GetVarint(&pData->p[i], (u64*)pnRow); + for(iCol=0; inn && iColp[i], (u64*)&anSize[iCol]); + } + } + + fts5DataRelease(pData); return fts5IndexReturn(p); } @@ -4993,18 +4791,25 @@ static void fts5TestTerm( if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; /* If this is a prefix query, check that the results returned if the - ** the index is disabled are the same. In both ASC and DESC order. */ - if( iIdx>0 && rc==SQLITE_OK ){ - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; - ck2 = 0; - rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; - } - if( iIdx>0 && rc==SQLITE_OK ){ - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; - ck2 = 0; - rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + ** the index is disabled are the same. In both ASC and DESC order. + ** + ** This check may only be performed if the hash table is empty. This + ** is because the hash table only supports a single scan query at + ** a time, and the multi-iter loop from which this function is called + ** is already performing such a scan. */ + if( p->nPendingData==0 ){ + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } } cksum3 ^= ck1; @@ -5043,16 +4848,65 @@ static void fts5IndexIntegrityCheckEmpty( /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ - Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ - if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; } fts5DataRelease(pLeaf); - if( p->rc ) break; } } +static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ + int iTermOff = 0; + int ii; + + Fts5Buffer buf1 = {0,0,0}; + Fts5Buffer buf2 = {0,0,0}; + + ii = pLeaf->szLeaf; + while( iinn && p->rc==SQLITE_OK ){ + int res; + int iOff; + int nIncr; + + ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); + iTermOff += nIncr; + iOff = iTermOff; + + if( iOff>=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else if( iTermOff==nIncr ){ + int nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + }else{ + int nKeep, nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + buf1.n = nKeep; + fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + + if( p->rc==SQLITE_OK ){ + res = fts5BufferCompare(&buf1, &buf2); + if( res<=0 ) p->rc = FTS5_CORRUPT; + } + } + fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); + } + + fts5BufferFree(&buf1); + fts5BufferFree(&buf2); +} + static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ @@ -5066,7 +4920,7 @@ static void fts5IndexIntegrityCheckSegment( if( pSeg->pgnoFirst==0 ) return; fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( - "SELECT segid, term, (pgno>>1), (pgno & 1) FROM '%q'.'%q_idx' WHERE segid=%d", + "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d", pConfig->zDb, pConfig->zName, pSeg->iSegid )); @@ -5074,7 +4928,6 @@ static void fts5IndexIntegrityCheckSegment( while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ - int iOff; /* Offset of first term on leaf */ int nIdxTerm = sqlite3_column_bytes(pStmt, 1); const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); @@ -5084,7 +4937,7 @@ static void fts5IndexIntegrityCheckSegment( /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ if( iIdxLeafpgnoFirst ) continue; - iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf); + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); pLeaf = fts5DataRead(p, iRow); if( pLeaf==0 ) break; @@ -5092,15 +4945,16 @@ static void fts5IndexIntegrityCheckSegment( ** to or larger than the split-key in zIdxTerm. Also check that if there ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ - iOff = fts5GetU16(&pLeaf->p[2]); - if( iOff==0 ){ + if( pLeaf->nn<=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ - int iRowidOff; + int iOff; /* Offset of first term on leaf */ + int iRowidOff; /* Offset of first rowid on leaf */ int nTerm; /* Size of term on leaf in bytes */ int res; /* Comparison of term and split-key */ - iRowidOff = fts5GetU16(&pLeaf->p[0]); + iOff = fts5LeafFirstTermOff(pLeaf); + iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff ){ p->rc = FTS5_CORRUPT; }else{ @@ -5109,6 +4963,8 @@ static void fts5IndexIntegrityCheckSegment( if( res==0 ) res = nTerm - nIdxTerm; if( res<0 ) p->rc = FTS5_CORRUPT; } + + fts5IntegrityCheckPgidx(p, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -5136,10 +4992,10 @@ static void fts5IndexIntegrityCheckSegment( /* Check any rowid-less pages that occur before the current leaf. */ for(iPg=iPrevLeaf+1; iPgp[0])!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; fts5DataRelease(pLeaf); } } @@ -5147,12 +5003,13 @@ static void fts5IndexIntegrityCheckSegment( /* Check that the leaf page indicated by the iterator really does ** contain the rowid suggested by the same. */ - iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPrevLeaf); + iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ i64 iRowid; - int iRowidOff = fts5GetU16(&pLeaf->p[0]); - if( iRowidOff>=pLeaf->n ){ + int iRowidOff = fts5LeafFirstRowidOff(pLeaf); + ASSERT_SZLEAF_OK(pLeaf); + if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); @@ -5327,13 +5184,13 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) "); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); }else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(structure)"); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(%ssegid=%d h=%d pgno=%d)", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno ); } @@ -5353,9 +5210,8 @@ static void fts5DebugStructure( ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, - " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, - pSeg->pgnoFirst, pSeg->pgnoLast + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); @@ -5413,11 +5269,13 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ ** The return value is the number of bytes read from the input buffer. */ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ - i64 iDocid; + i64 iDocid = 0; int iOff = 0; - iOff = sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDocid); - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid); + if( n>0 ){ + iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); + } while( iOff=4 ){ - iRowidOff = fts5GetU16(&a[0]); - iTermOff = fts5GetU16(&a[2]); - }else{ - sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt"); - goto decode_out; - } - - if( iRowidOff ){ - iOff = iRowidOff; - }else if( iTermOff ){ - iOff = iTermOff; - }else{ - iOff = n; - } - fts5DecodePoslist(&rc, &s, &a[4], iOff-4); - - assert( iRowidOff==0 || iOff==iRowidOff ); - if( iRowidOff ){ - iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff); - } - - assert( iTermOff==0 || iOff==iTermOff ); - while( iOffpStorage, nMerge); }else if( 0==sqlite3_stricmp("integrity-check", z) ){ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); +#ifdef SQLITE_DEBUG + }else if( 0==sqlite3_stricmp("prefix-index", z) ){ + pConfig->bPrefixIndex = sqlite3_value_int(pVal); +#endif }else{ rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); if( rc==SQLITE_OK ){ @@ -1498,11 +1502,13 @@ static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, - int (*xToken)(void*, const char*, int, int, int) + int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken); + return sqlite3Fts5Tokenize( + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -1655,13 +1661,16 @@ static int fts5ApiColumnText( static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ int *pCnt = (int*)pContext; - *pCnt = *pCnt + 1; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + (*pCnt)++; + } return SQLITE_OK; } @@ -1691,7 +1700,9 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ pCsr->aColumnSize[i] = 0; rc = fts5ApiColumnText(pCtx, i, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb); + rc = sqlite3Fts5Tokenize( + pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + ); } } } @@ -1853,7 +1864,7 @@ static int fts5ApiQueryPhrase( pNew->iFirstRowid = SMALLEST_INT64; pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; - rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); + rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ @@ -2344,7 +2355,7 @@ int sqlite3_fts5_init( void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 1; + pGlobal->api.iVersion = 2; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index f09b7d9158..65c6d48a36 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -359,6 +359,7 @@ struct Fts5InsertCtx { */ static int fts5StorageInsertCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ @@ -366,8 +367,10 @@ static int fts5StorageInsertCallback( ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - int iPos = pCtx->szCol++; - return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken); + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } /* @@ -394,6 +397,7 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ if( pConfig->abUnindexed[iCol-1] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pSeek, iCol), sqlite3_column_bytes(pSeek, iCol), (void*)&ctx, @@ -451,22 +455,7 @@ static int fts5StorageInsertDocsize( static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ int rc = SQLITE_OK; if( p->bTotalsValid==0 ){ - int nCol = p->pConfig->nCol; - Fts5Buffer buf; - memset(&buf, 0, sizeof(buf)); - - memset(p->aTotalSize, 0, sizeof(i64) * nCol); - p->nTotalRow = 0; - rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf); - if( rc==SQLITE_OK && buf.n ){ - int i = 0; - int iCol; - i += fts5GetVarint(&buf.p[i], (u64*)&p->nTotalRow); - for(iCol=0; iaTotalSize[iCol]); - } - } - sqlite3_free(buf.p); + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); p->bTotalsValid = bCache; } return rc; @@ -565,6 +554,7 @@ int sqlite3Fts5StorageSpecialDelete( if( pConfig->abUnindexed[iCol] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[iCol]), sqlite3_value_bytes(apVal[iCol]), (void*)&ctx, @@ -654,6 +644,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, ctx.iCol+1), sqlite3_column_bytes(pScan, ctx.iCol+1), (void*)&ctx, @@ -771,6 +762,7 @@ int sqlite3Fts5StorageInsert( ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), sqlite3_value_bytes(apVal[ctx.iCol+2]), (void*)&ctx, @@ -838,15 +830,18 @@ struct Fts5IntegrityCtx { */ static int fts5StorageIntegrityCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - int iPos = pCtx->szCol++; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ + pCtx->szCol++; + } pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken + pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken ); return SQLITE_OK; } @@ -881,19 +876,23 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; - rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - rc = sqlite3Fts5Tokenize( - pConfig, + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, i+1), sqlite3_column_bytes(pScan, i+1), (void*)&ctx, fts5StorageIntegrityCallback ); - if( ctx.szCol!=aColSize[i] ) rc = FTS5_CORRUPT; + if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + rc = FTS5_CORRUPT; + } aTotalSize[i] += ctx.szCol; } if( rc!=SQLITE_OK ) break; @@ -918,7 +917,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ rc = fts5StorageCount(p, "content", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pConfig->bColumnsize ){ i64 nRow; rc = fts5StorageCount(p, "docsize", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; @@ -1002,9 +1001,12 @@ static int fts5StorageDecodeSizeArray( ** otherwise. */ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ - int nCol = p->pConfig->nCol; - sqlite3_stmt *pLookup = 0; - int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + int nCol = p->pConfig->nCol; /* Number of user columns in table */ + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ + int rc; /* Return Code */ + + assert( p->pConfig->bColumnsize ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); if( rc==SQLITE_OK ){ int bCorrupt = 1; sqlite3_bind_int64(pLookup, 1, iRowid); diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 82f3e0390d..21ce5f7bde 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -141,6 +141,7 @@ struct F5tAuxData { static int xTokenizeCb( void *pCtx, + int tflags, const char *zToken, int nToken, int iStart, int iEnd ){ @@ -584,6 +585,7 @@ struct F5tTokenizeCtx { static int xTokenizeCb2( void *pCtx, + int tflags, const char *zToken, int nToken, int iStart, int iEnd ){ @@ -666,7 +668,9 @@ static int f5tTokenize( ctx.bSubst = (objc==5); ctx.pRet = pRet; ctx.zInput = zText; - rc = tokenizer.xTokenize(pTok, (void*)&ctx, zText, nText, xTokenizeCb2); + rc = tokenizer.xTokenize( + pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2 + ); tokenizer.xDelete(pTok); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0); @@ -688,11 +692,11 @@ static int f5tTokenize( typedef struct F5tTokenizerContext F5tTokenizerContext; typedef struct F5tTokenizerCb F5tTokenizerCb; typedef struct F5tTokenizerModule F5tTokenizerModule; -typedef struct F5tTokenizerModule F5tTokenizerInstance; +typedef struct F5tTokenizerInstance F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int); + int (*xToken)(void*, int, const char*, int, int, int); }; struct F5tTokenizerModule { @@ -701,6 +705,12 @@ struct F5tTokenizerModule { F5tTokenizerContext *pContext; }; +struct F5tTokenizerInstance { + Tcl_Interp *interp; + Tcl_Obj *pScript; + F5tTokenizerContext *pContext; +}; + static int f5tTokenizerCreate( void *pCtx, const char **azArg, @@ -748,26 +758,53 @@ static void f5tTokenizerDelete(Fts5Tokenizer *p){ static int f5tTokenizerTokenize( Fts5Tokenizer *p, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int, int, int) + int (*xToken)(void*, int, const char*, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; void *pOldCtx; - int (*xOldToken)(void*, const char*, int, int, int); + int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; int rc; + const char *zFlags; pOldCtx = pInst->pContext->pCtx; xOldToken = pInst->pContext->xToken; + pInst->pContext->pCtx = pCtx; + pInst->pContext->xToken = xToken; + + assert( + flags==FTS5_TOKENIZE_DOCUMENT + || flags==FTS5_TOKENIZE_AUX + || flags==FTS5_TOKENIZE_QUERY + || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) + ); pEval = Tcl_DuplicateObj(pInst->pScript); Tcl_IncrRefCount(pEval); - rc = Tcl_ListObjAppendElement( - pInst->interp, pEval, Tcl_NewStringObj(pText, nText) - ); - if( rc==TCL_OK ){ - rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); + switch( flags ){ + case FTS5_TOKENIZE_DOCUMENT: + zFlags = "document"; + break; + case FTS5_TOKENIZE_AUX: + zFlags = "aux"; + break; + case FTS5_TOKENIZE_QUERY: + zFlags = "query"; + break; + case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY): + zFlags = "prefixquery"; + break; + default: + assert( 0 ); + zFlags = "invalid"; + break; } + + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1)); + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText)); + rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); Tcl_DecrRefCount(pEval); pInst->pContext->pCtx = pOldCtx; @@ -776,7 +813,7 @@ static int f5tTokenizerTokenize( } /* -** sqlite3_fts5_token TEXT START END POS +** sqlite3_fts5_token ?-colocated? TEXT START END */ static int f5tTokenizerReturn( void * clientData, @@ -788,14 +825,29 @@ static int f5tTokenizerReturn( int iStart; int iEnd; int nToken; + int tflags = 0; char *zToken; int rc; - assert( p ); - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "TEXT START END"); + if( objc==5 ){ + int nArg; + char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); + if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ + tflags |= FTS5_TOKEN_COLOCATED; + }else{ + goto usage; + } + }else if( objc!=4 ){ + goto usage; + } + + zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken); + if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart) + || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd) + ){ return TCL_ERROR; } + if( p->xToken==0 ){ Tcl_AppendResult(interp, "sqlite3_fts5_token may only be used by tokenizer callback", 0 @@ -803,16 +855,13 @@ static int f5tTokenizerReturn( return TCL_ERROR; } - zToken = Tcl_GetStringFromObj(objv[1], &nToken); - if( Tcl_GetIntFromObj(interp, objv[2], &iStart) - || Tcl_GetIntFromObj(interp, objv[3], &iEnd) - ){ - return TCL_ERROR; - } - - rc = p->xToken(p->pCtx, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; + + usage: + Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END"); + return TCL_ERROR; } static void f5tDelTokenizer(void *pCtx){ diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 8ebf5f5077..355f23330d 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -352,7 +352,7 @@ static void fts5MatchinfoFunc( ){ const char *zArg; Fts5MatchinfoCtx *p; - int rc; + int rc = SQLITE_OK; if( nVal>0 ){ zArg = (const char*)sqlite3_value_text(apVal[0]); @@ -363,11 +363,16 @@ static void fts5MatchinfoFunc( p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); - pApi->xSetAuxdata(pFts, p, sqlite3_free); - if( p==0 ) return; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } } - rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + if( rc==SQLITE_OK ){ + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + } if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); }else{ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 426e35551b..29a4579b44 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -116,8 +116,9 @@ static void asciiFold(char *aOut, const char *aIn, int nByte){ static int fts5AsciiTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; int rc = SQLITE_OK; @@ -158,7 +159,7 @@ static int fts5AsciiTokenize( asciiFold(pFold, &pText[is], nByte); /* Invoke the token callback */ - rc = xToken(pCtx, pFold, nByte, is, ie); + rc = xToken(pCtx, 0, pFold, nByte, is, ie); is = ie+1; } @@ -385,8 +386,9 @@ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ static int fts5UnicodeTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; int rc = SQLITE_OK; @@ -475,7 +477,7 @@ static int fts5UnicodeTokenize( } /* Invoke the token callback */ - rc = xToken(pCtx, aFold, zOut-aFold, is, ie); + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); } tokenize_done: @@ -553,7 +555,7 @@ static int fts5PorterCreate( typedef struct PorterContext PorterContext; struct PorterContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int); + int (*xToken)(void*, int, const char*, int, int, int); char *aBuf; }; @@ -1118,6 +1120,7 @@ static void fts5PorterStep1A(char *aBuf, int *pnBuf){ static int fts5PorterCb( void *pCtx, + int tflags, const char *pToken, int nToken, int iStart, @@ -1175,10 +1178,10 @@ static int fts5PorterCb( nBuf--; } - return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd); + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); pass_through: - return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd); + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); } /* @@ -1187,8 +1190,9 @@ static int fts5PorterCb( static int fts5PorterTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; PorterContext sCtx; @@ -1196,7 +1200,7 @@ static int fts5PorterTokenize( sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb + p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb ); } @@ -1225,7 +1229,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ ); } - return SQLITE_OK; + return rc; } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 59058a8462..5eba1296ad 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -295,3 +295,36 @@ proc NOT {a b} { return $a } +#------------------------------------------------------------------------- +# This command is similar to [split], except that it also provides the +# start and end offsets of each token. For example: +# +# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8} +# + +proc gobble_whitespace {textvar} { + upvar $textvar t + regexp {([ ]*)(.*)} $t -> space t + return [string length $space] +} + +proc gobble_text {textvar wordvar} { + upvar $textvar t + upvar $wordvar w + regexp {([^ ]*)(.*)} $t -> w t + return [string length $w] +} + +proc fts5_tokenize_split {text} { + set token "" + set ret [list] + set iOff [gobble_whitespace text] + while {[set nToken [gobble_text text word]]} { + lappend ret $word $iOff [expr $iOff+$nToken] + incr iOff $nToken + incr iOff [gobble_whitespace text] + } + + set ret +} + diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index daa535cd9b..1d48e4f7d9 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -51,7 +51,7 @@ do_execsql_test 2.1 { do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } -} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} +} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/} foreach w {a b c d e f} { do_execsql_test 2.3.$w.asc { @@ -139,7 +139,6 @@ foreach {i x y} { #------------------------------------------------------------------------- # -breakpoint reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); @@ -201,6 +200,7 @@ for {set i 1} {$i <= 10} {incr i} { } execsql { INSERT INTO t1(t1) VALUES('integrity-check'); } } {} + if {[set_test_counter errors]} break } #------------------------------------------------------------------------- @@ -343,7 +343,7 @@ do_execsql_test 13.5 { } {1} do_execsql_test 13.6 { - SELECT rowid FROM t1 WHERE t1 MATCH '.'; + SELECT rowid FROM t1 WHERE t1 MATCH '""'; } {} #------------------------------------------------------------------------- @@ -506,6 +506,36 @@ do_execsql_test 18.3 { SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c } {1 1} +#-------------------------------------------------------------------- +# fts5 table in the temp schema. +# +reset_db +do_execsql_test 19.0 { + CREATE VIRTUAL TABLE temp.t1 USING fts5(x); + INSERT INTO t1 VALUES('x y z'); + INSERT INTO t1 VALUES('w x 1'); + SELECT rowid FROM t1 WHERE t1 MATCH 'x'; +} {1 2} + +#-------------------------------------------------------------------- +# Test that 6 and 7 byte varints can be read. +# +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE temp.tmp USING fts5(x); +} +set ::ids [list \ + 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] +] +do_test 20.1 { + foreach id $::ids { + execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') } + } + execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } +} $::ids + + + finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index b998db05ab..3881c7e161 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -205,6 +205,9 @@ foreach {T create} { return $ret } + do_execsql_test $T.integrity { + INSERT INTO t1(t1) VALUES('integrity-check'); + } foreach {bAsc sql} { 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 3c8ad253d1..6d7e39f793 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -90,13 +90,13 @@ foreach {tn q res} " do_test 1.6.$tn.1 { set n [execsql_reads $q] - puts -nonewline "(n=$n nReadX=$nReadX)" + #puts -nonewline "(n=$n nReadX=$nReadX)" expr {$n < ($nReadX / 8)} } {1} do_test 1.6.$tn.2 { set n [execsql_reads "$q ORDER BY rowid DESC"] - puts -nonewline "(n=$n nReadX=$nReadX)" + #puts -nonewline "(n=$n nReadX=$nReadX)" expr {$n < ($nReadX / 8)} } {1} diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index 99dfeb357b..efad1b2069 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -26,17 +26,17 @@ ifcapable !fts5 { do_execsql_test 1.1 { CREATE VIRTUAL TABLE ft1 USING fts5(x); SELECT * FROM ft1_config; -} {version 3} +} {version 4} do_execsql_test 1.2 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); SELECT * FROM ft1_config; -} {pgsz 32 version 3} +} {pgsz 32 version 4} do_execsql_test 1.3 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); SELECT * FROM ft1_config; -} {pgsz 64 version 3} +} {pgsz 64 version 4} #-------------------------------------------------------------------------- # Test the logic for parsing the rank() function definition. diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test index ed0edd677e..dec9b58d3d 100644 --- a/ext/fts5/test/fts5columnsize.test +++ b/ext/fts5/test/fts5columnsize.test @@ -134,5 +134,18 @@ do_execsql_test 3.2.1 { 1 {-1 0 -1} 2 {-1 0 -1} } +#------------------------------------------------------------------------- +# Test the integrity-check +# +do_execsql_test 4.1.1 { + CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0); + INSERT INTO t5 VALUES('1 2 3 4'); + INSERT INTO t5 VALUES('2 4 6 8'); +} + +breakpoint +do_execsql_test 4.1.2 { + INSERT INTO t5(t5) VALUES('integrity-check'); +} finish_test diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 3f57eb515a..edaafb2379 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -43,7 +43,7 @@ set segid [lindex [fts5_level_segids t1] 0] do_test 1.3 { execsql { - DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 0, 4); + DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} @@ -52,7 +52,7 @@ do_test 1.4 { db_restore_and_reopen execsql { UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE - rowid = fts5_rowid('segment', $segid, 0, 4); + rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 3e8323b984..3a4fcfaaed 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -209,13 +209,13 @@ foreach {tn nCut} { execsql ROLLBACK } - do_test 4.$tn.x { expr $nCorrupt>0 } 1 + # do_test 4.$tn.x { expr $nCorrupt>0 } 1 } } set doc [string repeat "A B C " 1000] -do_execsql_test 4.0 { +do_execsql_test 5.0 { CREATE VIRTUAL TABLE x5 USING fts5(tt); INSERT INTO x5(x5, rank) VALUES('pgsz', 32); WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) @@ -230,7 +230,7 @@ foreach {tn hdr} { foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] { if {$rowid & $mask} continue incr tn2 - do_test 4.$tn.$tn2 { + do_test 5.$tn.$tn2 { execsql BEGIN set fd [db incrblob main x5_data block $rowid] @@ -248,7 +248,7 @@ foreach {tn hdr} { #-------------------------------------------------------------------- reset_db -do_execsql_test 5.1 { +do_execsql_test 6.1 { CREATE VIRTUAL TABLE x5 USING fts5(tt); INSERT INTO x5 VALUES('a'); INSERT INTO x5 VALUES('a a'); @@ -262,7 +262,7 @@ proc colsize {cmd i} { } sqlite3_fts5_create_function db colsize colsize -do_catchsql_test 5.2 { +do_catchsql_test 6.2 { SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a' } {1 SQLITE_CORRUPT_VTAB} diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index cf08a5b107..ba1d6f9902 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -24,18 +24,23 @@ ifcapable !fts5 { } sqlite3_fts5_may_be_corrupt 1 +proc create_t1 {} { + expr srand(0) + db func rnddoc fts5_rnddoc + db eval { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) + INSERT INTO t1 SELECT rnddoc(10) FROM ii; + } +} + +if 1 { + # Create a simple FTS5 table containing 100 documents. Each document # contains 10 terms, each of which start with the character "x". # -expr srand(0) -db func rnddoc fts5_rnddoc -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x); - INSERT INTO t1(t1, rank) VALUES('pgsz', 64); - WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) - INSERT INTO t1 SELECT rnddoc(10) FROM ii; -} -set mask [expr 31 << 31] +do_test 1.0 { create_t1 } {} do_test 1.1 { # Pick out the rowid of the right-most b-tree leaf in the new segment. @@ -75,6 +80,261 @@ do_execsql_test 2.2 { SELECT length(block) FROM t2_data WHERE id=1; } {2} + +#------------------------------------------------------------------------- +# Test that missing leaf pages are recognized as corruption. +# +reset_db +do_test 3.0 { create_t1 } {} + +do_execsql_test 3.1 { + SELECT count(*) FROM t1_data; +} {105} + +proc do_3_test {tn} { + set i 0 + foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] { + incr i + do_test $tn.$i { + db eval BEGIN + db eval {DELETE FROM t1_data WHERE rowid = $::rowid} + list [ + catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg + ] $msg + } {1 {database disk image is malformed}} + catch { db eval ROLLBACK } + } +} + +do_3_test 3.2 + +do_execsql_test 3.3 { + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1(t1) VALUES('optimize'); +} {} + +do_3_test 3.4 + +do_test 3.5 { + execsql { + DELETE FROM t1; + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); + } + for {set i 0} {$i < 1000} {incr i} { + set rnd [expr int(rand() * 1000)] + set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]] + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } +} {} + +do_3_test 3.6 + +do_test 3.7 { + execsql { + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1(t1) VALUES('optimize'); + } +} {} + +do_3_test 3.8 + +do_test 3.9 { + execsql { + DELETE FROM t1; + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + for {set i 0} {$i < 100} {incr i} { + set rnd [expr int(rand() * 100)] + set doc "x[string repeat $rnd 20]" + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } +} {} + +do_3_test 3.10 + +#------------------------------------------------------------------------- +# Test that segments that end unexpectedly are identified as corruption. +# +reset_db +do_test 4.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + for {set i 0} {$i < 100} {incr i} { + set rnd [expr int(rand() * 100)] + set doc "x[string repeat $rnd 20]" + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } + execsql { INSERT INTO t1(t1) VALUES('optimize') } +} {} + +set nErr 0 +for {set i 1} {1} {incr i} { + set struct [db one {SELECT block FROM t1_data WHERE id=10}] + binary scan $struct c* var + set end [lindex $var end] + if {$end<=$i} break + lset var end [expr $end - $i] + set struct [binary format c* $var] + db eval { + BEGIN; + UPDATE t1_data SET block = $struct WHERE id=10; + } + do_test 4.1.$i { + incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }] + set {} {} + } {} + catch { db eval ROLLBACK } +} +do_test 4.1.x { expr $nErr>45 } 1 + +#------------------------------------------------------------------------- +# + +# The first argument passed to this command must be a binary blob +# containing an FTS5 leaf page. This command returns a copy of this +# blob, with the pgidx of the leaf page replaced by a single varint +# containing value $iVal. +# +proc rewrite_pgidx {blob iVal} { + binary scan $blob SS off1 szLeaf + if {$iVal<0 || $iVal>=128} { + error "$iVal out of range!" + } else { + set pgidx [binary format c $iVal] + } + + binary format a${szLeaf}a* $blob $pgidx +} + +reset_db +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 40); + BEGIN; + INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj'); + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1(x1) VALUES('optimize'); + COMMIT; +} + +#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b } +# +db func rewrite_pgidx rewrite_pgidx +set i 0 +foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] { + foreach val {2 100} { + do_test 5.2.$val.[incr i] { + catchsql { + BEGIN; + UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid; + SELECT rowid FROM x1 WHERE x1 MATCH 'xa*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xb*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xc*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xd*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xe*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xf*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xg*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xh*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xi*'; + } + set {} {} + } {} + catch { db eval ROLLBACK } + } +} + +} + +#------------------------------------------------------------------------ +# +reset_db +do_execsql_test 6.1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('bbbbb ccccc'); + SELECT quote(block) FROM t1_data WHERE rowid>100; +} {X'000000180630626262626201020201056363636363010203040A'} +do_execsql_test 6.1.1 { + UPDATE t1_data SET block = + X'000000180630626262626201020201056161616161010203040A' + WHERE rowid>100; +} +do_catchsql_test 6.1.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------- +reset_db +do_execsql_test 6.2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 VALUES('aa bb cc dd ee'); + SELECT pgno, quote(term) FROM t1_idx; +} {2 X'' 4 X'3064'} +do_execsql_test 6.2.1 { + UPDATE t1_idx SET term = X'3065' WHERE pgno=4; +} +do_catchsql_test 6.2.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + +#------- +reset_db +do_execsql_test 6.3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('abc abcdef abcdefghi'); + SELECT quote(block) FROM t1_data WHERE id>100; +} {X'0000001C043061626301020204036465660102030703676869010204040808'} +do_execsql_test 6.3.1 { + BEGIN; + UPDATE t1_data SET block = + X'0000001C043061626301020204036465660102035003676869010204040808' + ------------------------------------------^^--------------------- + WHERE id>100; +} +do_catchsql_test 6.3.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 6.3.3 { + ROLLBACK; + BEGIN; + UPDATE t1_data SET block = + X'0000001C043061626301020204036465660102030750676869010204040808' + --------------------------------------------^^------------------- + WHERE id>100; +} +do_catchsql_test 6.3.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 6.3.4 { + ROLLBACK; + BEGIN; + UPDATE t1_data SET block = + X'0000001C043061626301020204036465660102030707676869010204040850' + --------------------------------------------------------------^^- + WHERE id>100; +} +do_catchsql_test 6.3.5 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} +do_execsql_test 6.3.6 { + ROLLBACK; + BEGIN; + UPDATE t1_data SET block = + X'0000001C503061626301020204036465660102030707676869010204040808' + ----------^^----------------------------------------------------- + WHERE id>100; +} +do_catchsql_test 6.3.5 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {1 {database disk image is malformed}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 07d7e2baeb..5a03c6989b 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -26,6 +26,8 @@ if { $tcl_platform(wordSize)<8 } { return } +if 1 { + proc do_fb_test {tn sql res} { set res2 [lsort -integer -decr $res] uplevel [list do_execsql_test $tn.1 $sql $res] @@ -128,5 +130,68 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} { do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] +} + +#-------------------------------------------------------------------- +# +reset_db + +set ::vocab [list \ + IteratorpItercurrentlypointstothefirstrowidofadoclist \ + Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ + pageIfthecurrenttermisthelasttermonthepageloadthe \ + doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \ + IteratorpItercurrentlypointstothefirstrowidofadoclist \ + Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ + pageIfthecurrenttermisthelasttermonthepageloadthe \ + doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \ +] +proc rnddoc {} { + global vocab + set nVocab [llength $vocab] + set ret [list] + for {set i 0} {$i < 64} {incr i} { + lappend ret [lindex $vocab [expr $i % $nVocab]] + } + set ret +} +db func rnddoc rnddoc + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE abc USING fts5(a); + INSERT INTO abc(abc, rank) VALUES('pgsz', 32); + + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + + INSERT INTO abc SELECT rnddoc() FROM abc; + INSERT INTO abc SELECT rnddoc() FROM abc; +} + + + +do_execsql_test 3.2 { + SELECT rowid FROM abc WHERE abc + MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' + ORDER BY rowid DESC; +} {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} + +do_execsql_test 3.2 { + INSERT INTO abc(abc) VALUES('integrity-check'); + INSERT INTO abc(abc) VALUES('optimize'); + INSERT INTO abc(abc) VALUES('integrity-check'); +} + +set v [lindex $vocab 0] +set i 0 +foreach v $vocab { + do_execsql_test 3.3.[incr i] { + SELECT rowid FROM abc WHERE abc MATCH $v + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} +} + + finish_test diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test index ad05412ba9..3ccbd7d7a2 100644 --- a/ext/fts5/test/fts5ea.test +++ b/ext/fts5/test/fts5ea.test @@ -87,6 +87,12 @@ do_execsql_test 4.0 { SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"'); } {{"a" AND """"}} +#------------------------------------------------------------------------- +# Experiment with a tokenizer that considers " to be a token character. +# +do_catchsql_test 5.0 { + SELECT fts5_expr('abc | def'); +} {1 {fts5: syntax error near "|"}} diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 352e1b4a17..55c4b15cf3 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -30,18 +30,18 @@ proc do_syntax_test {tn expr res} { } foreach {tn expr res} { - 1 {abc} {"abc"} - 2 {abc .} {"abc"} - 3 {.} {} - 4 {abc OR .} {"abc"} - 5 {abc NOT .} {"abc"} - 6 {abc AND .} {"abc"} - 7 {. OR abc} {"abc"} - 8 {. NOT abc} {"abc"} - 9 {. AND abc} {"abc"} - 10 {abc + . + def} {"abc" + "def"} - 11 {abc . def} {"abc" AND "def"} - 12 {r+e OR w} {"r" + "e" OR "w"} + 1 {abc} {"abc"} + 2 {abc ""} {"abc"} + 3 {""} {} + 4 {abc OR ""} {"abc"} + 5 {abc NOT ""} {"abc"} + 6 {abc AND ""} {"abc"} + 7 {"" OR abc} {"abc"} + 8 {"" NOT abc} {"abc"} + 9 {"" AND abc} {"abc"} + 10 {abc + "" + def} {"abc" + "def"} + 11 {abc "" def} {"abc" AND "def"} + 12 {r+e OR w} {"r" + "e" OR "w"} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b9657be1cc..c79cf7ab70 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -22,6 +22,7 @@ ifcapable !fts5 { return } + #------------------------------------------------------------------------- # OOM while rebuilding an FTS5 table. # @@ -148,5 +149,149 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +# +# 5.2.* OOM while running a query that includes synonyms and matchinfo(). +# +# 5.3.* OOM while running a query that returns a row containing instances +# of more than 4 synonyms for a single term. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i < 7} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} +proc tcl_create {args} { return "tcl_tokenize" } +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +db func mit mit +sqlite3_fts5_register_matchinfo db +do_test 5.0 { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } + foreach {rowid text} { + 1 {aaaa cc b aaaaa cc aa} + 2 {aa aa bb a bbb} + 3 {bb aaaaa aaaaa b aaaa aaaaa} + 4 {aa a b aaaa aa} + 5 {aa b ccc aaaaa cc} + 6 {aa aaaaa bbbb cc aaa} + 7 {aaaaa aa aa ccccc bb} + 8 {ccc bbbbb ccccc bbb c} + 9 {cccccc bbbb a aaa cccc c} + + 20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee} + 21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e} + 22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd} + 23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff} + 24 {eee dd ee dddd dddd eeeeee e eee fff ffff} + 25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f} + 26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee} + 27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d} + 28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd} + 29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee} + 30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee} + 31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd} + 32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff} + 33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd} + 34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff} + 35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff} + 36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff} + 37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee} + 38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee} + 39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd} + 40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f} + 41 {ddd ddd fff fffff ee fffff f fff ddddd fffff} + 42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee} + 43 {d dd fff fffff d f fff e dddd ee ee} + 44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee} + 45 {eeee f eeeee ee e ffff f ddd e fff} + 46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee} + 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} + 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} + 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} + + 50 {x e} + } { + execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } + } +} {} + +set res [list {*}{ + 1 {3 24 8 2 12 6} + 5 {2 24 8 2 12 6} + 6 {3 24 8 1 12 6} + 7 {3 24 8 1 12 6} + 9 {2 24 8 3 12 6} +}] +do_execsql_test 5.1.1 { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' +} $res +do_execsql_test 5.1.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f' +} 29 + +faultsim_save_and_close +do_faultsim_test 5.2 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create + sqlite3_fts5_register_matchinfo db + db func mit mit +} -body { + db eval { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' + } +} -test { + faultsim_test_result [list 0 $::res] +} + +do_faultsim_test 5.3 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f' + } +} -test { + faultsim_test_result {0 29} +} + +do_faultsim_test 5.4 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e' + } +} -test { + faultsim_test_result {0 1} +} + +#------------------------------------------------------------------------- +catch { db close } +breakpoint +do_faultsim_test 6 -faults oom* -prep { + sqlite_orig db test.db + sqlite3_db_config_lookaside db 0 0 0 +} -body { + load_static_extension db fts5 +} -test { + faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} + if {$testrc==0} { + db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) } + } + db close +} finish_test diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test new file mode 100644 index 0000000000..9bce1c7caf --- /dev/null +++ b/ext/fts5/test/fts5fault7.test @@ -0,0 +1,91 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault7 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if 1 { + +#------------------------------------------------------------------------- +# Test fault-injection on a query that uses xColumnSize() on columnsize=0 +# table. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0); + INSERT INTO t1 VALUES('a b c d e f g'); + INSERT INTO t1 VALUES('a b c d'); + INSERT INTO t1 VALUES('a b c d e f g h i j'); +} + + +fts5_aux_test_functions db +do_faultsim_test 1 -faults oom* -body { + execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' } +} -test { + faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM} +} + +} + +#------------------------------------------------------------------------- +# Test fault-injection when a segment is promoted. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a); + INSERT INTO t2(t2, rank) VALUES('automerge', 0); + INSERT INTO t2(t2, rank) VALUES('crisismerge', 4); + INSERT INTO t2(t2, rank) VALUES('pgsz', 40); + + INSERT INTO t2 VALUES('a b c'); + INSERT INTO t2 VALUES('d e f'); + INSERT INTO t2 VALUES('f e d'); + INSERT INTO t2 VALUES('c b a'); + + INSERT INTO t2 VALUES('a b c'); + INSERT INTO t2 VALUES('d e f'); + INSERT INTO t2 VALUES('f e d'); + INSERT INTO t2 VALUES('c b a'); +} {} + +faultsim_save_and_close +do_faultsim_test 1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO t2 VALUES('c d c g g f'); + INSERT INTO t2 VALUES('c d g b f d'); + INSERT INTO t2 VALUES('c c f d e d'); + INSERT INTO t2 VALUES('e a f c e f'); + INSERT INTO t2 VALUES('c g f b b d'); + INSERT INTO t2 VALUES('d a g a b b'); + INSERT INTO t2 VALUES('e f a b c e'); + INSERT INTO t2 VALUES('e c a g c d'); + INSERT INTO t2 VALUES('g b d d e b'); + INSERT INTO t2 VALUES('e a d a e d'); + } +} -body { + db eval COMMIT +} -test { + faultsim_test_result {0 {}} +} + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 359702eff6..21f9b003e7 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -355,10 +355,10 @@ do_execsql_test 10.1 { #--------------------------------------------------------------------------- # Test the 'y' matchinfo flag # -set sqlite_fts3_enable_parentheses 1 reset_db +sqlite3_fts5_register_matchinfo db do_execsql_test 11.0 { - CREATE VIRTUAL TABLE tt USING fts3(x, y); + CREATE VIRTUAL TABLE tt USING fts5(x, y); INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 @@ -432,19 +432,18 @@ foreach {tn expr res} { SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr } $r2 } -set sqlite_fts3_enable_parentheses 0 #--------------------------------------------------------------------------- # Test the 'b' matchinfo flag # -set sqlite_fts3_enable_parentheses 1 reset_db +sqlite3_fts5_register_matchinfo db db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } - execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])" } {} do_execsql_test 12.1 { @@ -452,6 +451,5 @@ do_execsql_test 12.1 { SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] -set sqlite_fts3_enable_parentheses 0 finish_test diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test index 453d79867b..621934c0f2 100644 --- a/ext/fts5/test/fts5rowid.test +++ b/ext/fts5/test/fts5rowid.test @@ -27,15 +27,15 @@ do_catchsql_test 1.1 { do_catchsql_test 1.2 { SELECT fts5_rowid('segment') -} {1 {should be: fts5_rowid('segment', segid, height, pgno))}} +} {1 {should be: fts5_rowid('segment', segid, pgno))}} do_execsql_test 1.3 { - SELECT fts5_rowid('segment', 1, 1, 1) -} {139586437121} + SELECT fts5_rowid('segment', 1, 1) +} {137438953473} do_catchsql_test 1.4 { SELECT fts5_rowid('nosucharg'); -} {1 {first arg to fts5_rowid() must be 'segment' or 'start-of-index'}} +} {1 {first arg to fts5_rowid() must be 'segment'}} #------------------------------------------------------------------------- @@ -91,6 +91,10 @@ do_execsql_test 2.6 { # SELECT count(fts5_decode(rowid, block)) FROM x1_data; #} $res +do_execsql_test 2.8 { + SELECT fts5_decode(fts5_rowid('segment', 1000, 1), X'AB') +} {corrupt} + #------------------------------------------------------------------------- # Tests with very large tokens. # diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test new file mode 100644 index 0000000000..c93519e6b9 --- /dev/null +++ b/ext/fts5/test/fts5simple.test @@ -0,0 +1,173 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if 1 { +#------------------------------------------------------------------------- +# +set doc "x x [string repeat {y } 50]z z" +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO t1 VALUES($doc); + COMMIT; +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('d e f'); + INSERT INTO t1(t1) VALUES('optimize'); +} + +do_execsql_test 2.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, prefix='1,2'); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO t1 VALUES('one'); + SELECT * FROM t1 WHERE t1 MATCH 'o*'; +} {one} + +do_execsql_test 3.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t11 USING fts5(content); + INSERT INTO t11(t11, rank) VALUES('pgsz', 32); + INSERT INTO t11 VALUES('another'); + INSERT INTO t11 VALUES('string'); + INSERT INTO t11 VALUES('of'); + INSERT INTO t11 VALUES('text'); +} +do_test 4.2 { + execsql { INSERT INTO t11(t11) VALUES('optimize') } +} {} +do_execsql_test 4.3 { + INSERT INTO t11(t11) VALUES('integrity-check'); +} {} + +#db eval { SELECT fts5_decode(rowid, block) as x FROM t11_data } { puts $x } + +#------------------------------------------------------------------------- +reset_db +set doc [string repeat "x y " 5] +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE yy USING fts5(content); + INSERT INTO yy(yy, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + COMMIT; +} + +do_execsql_test 5.2 { + SELECT rowid FROM yy WHERE yy MATCH 'y' ORDER BY rowid ASC +} {1 2 3 4 5 6 7 8} + +do_execsql_test 5.3 { + SELECT rowid FROM yy WHERE yy MATCH 'y' ORDER BY rowid DESC +} {8 7 6 5 4 3 2 1} + +#db eval { SELECT fts5_decode(rowid, block) as x FROM yy_data } { puts $x } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE tt USING fts5(content); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + INSERT INTO tt VALUES('aa'); +} + +do_execsql_test 5.2 { + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1} + +do_execsql_test 5.3 { + DELETE FROM tt; + BEGIN; + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + COMMIT; +} {} + +do_execsql_test 5.4 { + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1 2} + +} + +do_execsql_test 5.5 { + DELETE FROM tt; + BEGIN; + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + INSERT INTO tt VALUES('aa'); + INSERT INTO tt VALUES('ab'); + COMMIT; + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1 2 3 4 5 6 7 8} + +do_execsql_test 5.6 { + INSERT INTO tt(tt) VALUES('integrity-check'); +} + +reset_db +do_execsql_test 5.7 { + CREATE VIRTUAL TABLE tt USING fts5(content); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + INSERT INTO tt VALUES('aa ab ac ad ae af'); +} + +do_execsql_test 5.8 { + SELECT rowid FROM tt WHERE tt MATCH 'a*'; +} {1} + +finish_test + diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test new file mode 100644 index 0000000000..008d2fc543 --- /dev/null +++ b/ext/fts5/test/fts5synonym.test @@ -0,0 +1,460 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach S { + {zero 0} + {one 1 i} + {two 2 ii} + {three 3 iii} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} +} { + foreach s $S { + set o [list] + foreach x $S {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +proc tcl_create {args} { + return "tcl_tokenize" +} + +sqlite3_fts5_create_tokenizer db tcl tcl_create + +#------------------------------------------------------------------------- +# Warm body test for the code in fts5_tcl.c. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('abc def ghi'); + INSERT INTO ft VALUES('jkl mno pqr'); + SELECT rowid, x FROM ft WHERE ft MATCH 'def'; + SELECT x, rowid FROM ft WHERE ft MATCH 'pqr'; +} {1 {abc def ghi} {jkl mno pqr} 2} + +#------------------------------------------------------------------------- +# Test a tokenizer that supports synonyms by adding extra entries to the +# FTS index. +# + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="document" && [info exists ::syn($w)]} { + foreach s $::syn($w) { + sqlite3_fts5_token -colo $s $iStart $iEnd + } + } + } +} +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + INSERT INTO ft VALUES('four five six'); + INSERT INTO ft VALUES('eight nine ten'); +} {} + +foreach {tn expr res} { + 1 "3" 1 + 2 "eight OR 8 OR 5" {2 3} + 3 "10" {} + 4 "1*" {1} + 5 "1 + 2" {1} +} { + do_execsql_test 2.1.$tn { + SELECT rowid FROM ft WHERE ft MATCH $expr + } $res +} + +#------------------------------------------------------------------------- +# Test some broken tokenizers: +# +# 3.1.*: A tokenizer that declares the very first token to be colocated. +# +# 3.2.*: A tokenizer that reports two identical tokens at the same position. +# This is allowed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + set bColo 1 + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + if {$bColo} { + sqlite3_fts5_token -colo $w $iStart $iEnd + set bColo 0 + } { + sqlite3_fts5_token $w $iStart $iEnd + } + } +} +do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 1 three 1 1 two 1 1 +} + +do_execsql_test 3.1.1 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} {} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 3.1.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three' +} {1} + +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + sqlite3_fts5_token -colo $w $iStart $iEnd + } +} +do_execsql_test 3.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 4 three 1 2 two 1 2 +} +do_execsql_test 3.2.1 { + SELECT rowid FROM ft WHERE ft MATCH 'one'; +} {1} +do_execsql_test 3.2.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three'; +} {1} +do_execsql_test 3.2.3 { + SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; +} {1} +do_execsql_test 3.2.4 { + SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; +} {1} +do_execsql_test 3.2.5 { + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {} + +#------------------------------------------------------------------------- +# Check that expressions with synonyms can be parsed and executed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [info exists ::syn($w)]} { + foreach s $::syn($w) { + sqlite3_fts5_token -colo $s $iStart $iEnd + } + } + } +} + +foreach {tn expr res} { + 1 {abc} {"abc"} + 2 {one} {"one"|"i"|"1"} + 3 {3} {"3"|"iii"|"three"} + 4 {3*} {"3"|"iii"|"three" *} +} { + do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] +} + +do_execsql_test 4.2.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl); + INSERT INTO xx VALUES('one two'); + INSERT INTO xx VALUES('three four'); +} + +do_execsql_test 4.2.2 { + SELECT rowid FROM xx WHERE xx MATCH '2' +} {1} + +do_execsql_test 4.2.3 { + SELECT rowid FROM xx WHERE xx MATCH '3' +} {2} + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {four v 4 i three} {1 3 five five 4 one} + 2 {5 1 3 4 i} {2 2 v two 4} + 3 {5 i 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one 5 1 iii 4 3} + 5 {three i v i four 4 1} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv} + 7 {ii ii two three 2 5} {iii i ii iii iii one one} + 8 {2 ii i two 3 three 2} {two iv v iii 3 five} + 9 {i 2 iv 3 five four v} {iii 4 three i three ii 1} + } { + execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + + +foreach {tn q res} { + 1 {one} { + 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} + 2 {5 [1] 3 4 [i]} {2 2 v two 4} + 3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]} + 4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3} + 5 {three [i] v [i] four 4 [1]} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv} + 7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]} + 8 {2 ii [i] two 3 three 2} {two iv v iii 3 five} + 9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]} + } + 2 {five four} { + 1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one} + 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} + 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} + 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} + 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} + 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} + 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} + } + 3 {one OR two OR iii OR 4 OR v} { + 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]} + 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]} + 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]} + 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]} + 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]} + 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]} + 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]} + 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} + 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} + } + + 4 {5 + 1} { + 2 {[5 1] 3 4 i} {2 2 v two 4} + 3 {[5 i] 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one [5 1] iii 4 3} + 5 {three i [v i] four 4 1} {ii five five five iii} + } + + 5 {one + two + three} { + 7 {ii ii two three 2 5} {iii [i ii iii] iii one one} + 8 {2 ii [i two 3] three 2} {two iv v iii 3 five} + } + + 6 {"v v"} { + 1 {four v 4 i three} {1 3 [five five] 4 one} + 5 {three i v i four 4 1} {ii [five five five] iii} + } +} { + do_execsql_test 5.1.$tn { + SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') + FROM t1 WHERE t1 MATCH $q + } $res +} + +# Test that the xQueryPhrase() API works with synonyms. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit +sqlite3_fts5_register_matchinfo db + +foreach {tn q res} { + 1 {one} { + 1 {1 11 7 2 12 6} 2 {2 11 7 0 12 6} + 3 {2 11 7 1 12 6} 4 {1 11 7 2 12 6} + 5 {3 11 7 0 12 6} 6 {0 11 7 2 12 6} + 7 {0 11 7 3 12 6} 8 {1 11 7 0 12 6} + 9 {1 11 7 2 12 6} + } +} { + do_execsql_test 5.2.$tn { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q + } $res +} + + +#------------------------------------------------------------------------- +# Test terms with more than 4 synonyms. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_execsql_test 6.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl); + INSERT INTO t1 VALUES('yy xx qq'); + INSERT INTO t1 VALUES('yy xx xx'); +} +do_execsql_test 6.0.2 { + SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)'; +} {{yy xx qq}} + +do_test 6.0.3 { + execsql { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii} + 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq} + 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu} + 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss} + 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } { + execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {a} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]} + 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq} + } + + 2 {a AND q} { + 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]} + } + + 3 {o OR (q AND a)} { + 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} + 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } + + 4 {NEAR(q y, 20)} { + 1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} + } +} { + do_execsql_test 6.1.$tn.asc { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q + } $res + + set res2 [list] + foreach {rowid a b} $res { + set res2 [concat [list $rowid $a $b] $res2] + } + + do_execsql_test 6.1.$tn.desc { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q ORDER BY rowid DESC + } $res2 +} + +do_execsql_test 6.2.1 { + INSERT INTO t2(rowid, a, b) VALUES(13, + 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy' + ); + SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')') + FROM t2 WHERE t2 MATCH 'x OR y' +} { + 1 { vvvvv qq oo vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq} + 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii} + 13 { } + {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} +} + +#------------------------------------------------------------------------- +# Test that the xColumnSize() API is not confused by colocated tokens. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_aux_test_functions db +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {[string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_execsql_test 7.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl); + INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} + +do_execsql_test 7.0.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl); + INSERT INTO t2 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} + +do_execsql_test 7.1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +finish_test + diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 8c5a772146..7e4d74d114 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -30,34 +30,34 @@ do_execsql_test 1.1 { do_execsql_test 1.2 { SELECT * FROM t1_config WHERE k='version' -} {version 3} +} {version 4} do_execsql_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'a'; } {1} do_execsql_test 1.4 { - UPDATE t1_config set v=4 WHERE k='version'; + UPDATE t1_config set v=5 WHERE k='version'; } do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} do_test 1.6 { db close sqlite3 db test.db catchsql { INSERT INTO t1 VALUES('x y z') } -} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} do_test 1.7 { execsql { DELETE FROM t1_config WHERE k='version' } db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 0, expected 3) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}} finish_test diff --git a/ext/fts5/tool/loadfts5.tcl b/ext/fts5/tool/loadfts5.tcl index 048de3ccd9..4bf89d7817 100644 --- a/ext/fts5/tool/loadfts5.tcl +++ b/ext/fts5/tool/loadfts5.tcl @@ -19,6 +19,12 @@ proc load_hierachy {dir} { db eval { INSERT INTO t1 VALUES($f, loadfile($f)) } incr ::nRow + if {$::O(trans) && ($::nRow % $::O(trans))==0} { + db eval { COMMIT } + db eval { INSERT INTO t1(t1) VALUES('integrity-check') } + db eval { BEGIN } + } + if {($::nRow % $::nRowPerDot)==0} { puts -nonewline . if {($::nRow % (65*$::nRowPerDot))==0} { puts "" } @@ -41,6 +47,7 @@ proc usage {} { puts stderr " -automerge N (set the automerge parameter to N)" puts stderr " -crisismerge N (set the crisismerge parameter to N)" puts stderr " -prefix PREFIX (comma separated prefix= argument)" + puts stderr " -trans N (commit after N inserts - 0 == never)" exit 1 } @@ -51,6 +58,7 @@ set O(delete) 0 set O(automerge) -1 set O(crisismerge) -1 set O(prefix) "" +set O(trans) 0 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] @@ -77,6 +85,11 @@ for {set i 0} {$i < $nOpt} {incr i} { if { [incr i]>=$nOpt } usage set O(limit) [lindex $argv $i] } + + -trans { + if { [incr i]>=$nOpt } usage + set O(trans) [lindex $argv $i] + } -automerge { if { [incr i]>=$nOpt } usage @@ -104,8 +117,9 @@ if {$O(delete)} { file delete -force $dbfile } sqlite3 db $dbfile catch { load_static_extension db fts5 } db func loadfile loadfile +db eval "PRAGMA page_size=4096" -db transaction { +db eval BEGIN set pref "" if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } catch { @@ -126,7 +140,7 @@ db transaction { } } load_hierachy [lindex $argv end] -} +db eval COMMIT diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl index 846902b3be..d9af5f38eb 100644 --- a/ext/fts5/tool/showfts5.tcl +++ b/ext/fts5/tool/showfts5.tcl @@ -5,14 +5,52 @@ # Process command line arguments. # proc usage {} { - puts stderr "usage: $::argv0 database table" + puts stderr "usage: $::argv0 ?OPTIONS? database table" + puts stderr "" + puts stderr " -nterm (count number of terms in each segment)" puts stderr "" exit 1 } -if {[llength $argv]!=2} usage -set database [lindex $argv 0] -set tbl [lindex $argv 1] +set O(nterm) 0 + +if {[llength $argv]<2} usage +foreach a [lrange $argv 0 end-2] { + switch -- $a { + -nterm { + set O(nterm) 1 + } + + default { + usage + } + } +} + +set database [lindex $argv end-1] +set tbl [lindex $argv end] + + +#------------------------------------------------------------------------- +# Count the number of terms in each segment of fts5 table $tbl. Store the +# counts in the array variable in the parent context named by parameter +# $arrayname, indexed by segment-id. Example: +# +# count_terms fts_tbl A +# foreach {k v} [array get A] { puts "segid=$k nTerm=$v" } +# +proc count_terms {tbl arrayname} { + upvar A $arrayname + array unset A + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data" { + set desc [lindex $d 0] + if {[regexp {^segid=([0-9]*)} $desc -> id]} { + foreach i [lrange $d 1 end] { + if {[string match {term=*} $i]} { incr A($id) } + } + } + } +} #------------------------------------------------------------------------- @@ -21,11 +59,21 @@ set tbl [lindex $argv 1] sqlite3 db $database catch { load_static_extension db fts5 } +if {$O(nterm)} { count_terms $tbl A } + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" { foreach lvl [lrange $d 1 end] { puts [lrange $lvl 0 2] + foreach seg [lrange $lvl 3 end] { - puts " $seg" + if {$::O(nterm)} { + regexp {^id=([0-9]*)} $seg -> id + set nTerm 0 + catch { set nTerm $A($id) } + puts [format " % -28s nTerm=%d" $seg $nTerm] + } else { + puts [format " % -28s" $seg] + } } } } diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 5df7551dec..ca6021733c 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -68,6 +68,9 @@ struct JsonString { #define JSON_ARRAY 6 #define JSON_OBJECT 7 +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + /* ** Names of the various JSON types: */ @@ -82,7 +85,7 @@ static const char * const jsonType[] = { #define JNODE_REMOVE 0x04 /* Do not output */ #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ #define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_JSON 0x20 /* Treat REPLACE as JSON text */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ /* A single node of parsed JSON @@ -242,8 +245,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ */ static void jsonAppendValue( JsonString *p, /* Append to this JSON string */ - sqlite3_value *pValue, /* Value to append */ - u8 textIsJson /* Try to treat text values as JSON */ + sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { @@ -260,7 +262,7 @@ static void jsonAppendValue( case SQLITE_TEXT: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); - if( textIsJson ){ + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ jsonAppendRaw(p, z, n); }else{ jsonAppendString(p, z, n); @@ -364,8 +366,7 @@ static void jsonRenderNode( if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ if( pNode[j].jnFlags & JNODE_REPLACE ){ jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal], - (pNode[j].jnFlags & JNODE_JSON)!=0); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); } }else{ jsonAppendSeparator(pOut); @@ -390,8 +391,7 @@ static void jsonRenderNode( jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal], - (pNode[j+1].jnFlags & JNODE_JSON)!=0); + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); }else{ jsonRenderNode(&pNode[j+1], pOut, aReplace); } @@ -420,6 +420,7 @@ static void jsonReturnJson( jsonInit(&s, pCtx); jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } /* @@ -494,12 +495,12 @@ static void jsonReturn( } if( v==0 ) break; if( v<=0x7f ){ - zOut[j++] = v; + zOut[j++] = (char)v; }else if( v<=0x7ff ){ - zOut[j++] = 0xc0 | (v>>6); + zOut[j++] = (char)(0xc0 | (v>>6)); zOut[j++] = 0x80 | (v&0x3f); }else{ - zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = (char)(0xe0 | (v>>12)); zOut[j++] = 0x80 | ((v>>6)&0x3f); zOut[j++] = 0x80 | (v&0x3f); } @@ -583,6 +584,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ u32 j; int iThis; int x; + JsonNode *pNode; while( isspace(pParse->zJson[i]) ){ i++; } if( (c = pParse->zJson[i])==0 ) return 0; if( c=='{' ){ @@ -597,7 +599,9 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return -1; } if( pParse->oom ) return -1; - if( pParse->aNode[pParse->nNode-1].eType!=JSON_STRING ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; j = x; while( isspace(pParse->zJson[j]) ){ j++; } if( pParse->zJson[j]!=':' ) return -1; @@ -947,21 +951,15 @@ static char *jsonPathSyntaxError(const char *zErr){ ** ** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if ** nodes are appended. -** -** If the path starts with $$ then set *pFlags to JNODE_REPLACE|JNODE_JSON -** as a single to the caller that the input text to be inserted should be -** interpreted as JSON rather than as ordinary text. */ static JsonNode *jsonLookup( JsonParse *pParse, /* The JSON to search */ const char *zPath, /* The path to search */ int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx, /* Report errors here, if not NULL */ - u8 *pFlags /* Write JNODE_REPLACE or _REPLACE|_JSON here */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ ){ const char *zErr = 0; JsonNode *pNode = 0; - u8 fg = JNODE_REPLACE; if( zPath==0 ) return 0; if( zPath[0]!='$' ){ @@ -969,15 +967,6 @@ static JsonNode *jsonLookup( goto lookup_err; } zPath++; - if( zPath[0]=='$' ){ - if( pFlags==0 ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - fg = JNODE_REPLACE|JNODE_JSON; - } - if( pFlags ) *pFlags = fg; pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); return pNode; @@ -992,7 +981,6 @@ lookup_err: sqlite3_result_error_nomem(pCtx); } } - if( pFlags ) *pFlags = fg; return 0; } @@ -1036,45 +1024,37 @@ static void jsonParseFunc( jsonParseFindParents(&x); jsonInit(&s, ctx); for(i=0; i2 ){ jsonAppendSeparator(&jx); @@ -1185,6 +1166,7 @@ static void jsonExtractFunc( if( argc>2 && i==argc ){ jsonAppendChar(&jx, ']'); jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } jsonReset(&jx); jsonParseReset(&x); @@ -1223,10 +1205,11 @@ static void jsonObjectFunc( n = (u32)sqlite3_value_bytes(argv[i]); jsonAppendString(&jx, z, n); jsonAppendChar(&jx, ':'); - jsonAppendValue(&jx, argv[i+1], 0); + jsonAppendValue(&jx, argv[i+1]); } jsonAppendChar(&jx, '}'); jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -1252,7 +1235,7 @@ static void jsonRemoveFunc( for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; - pNode = jsonLookup(&x, zPath, 0, ctx, 0); + pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto remove_done; if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } @@ -1288,14 +1271,12 @@ static void jsonReplaceFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ - u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx, &jnFlags); + pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto replace_err; if( pNode ){ - pNode->jnFlags &= ~JNODE_JSON; - pNode->jnFlags |= jnFlags; - pNode->iVal = i+1; + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->iVal = (u8)(i+1); } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ @@ -1340,19 +1321,17 @@ static void jsonSetFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ - u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx, &jnFlags); + pNode = jsonLookup(&x, zPath, &bApnd, ctx); if( x.oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; }else if( x.nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags &= ~JNODE_JSON; - pNode->jnFlags |= jnFlags; - pNode->iVal = i+1; + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->iVal = (u8)(i+1); } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ @@ -1385,7 +1364,7 @@ static void jsonTypeFunc( JsonNode *pNode; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(&x, zPath, 0, ctx, 0); + pNode = jsonLookup(&x, zPath, 0, ctx); }else{ pNode = x.aNode; } @@ -1410,6 +1389,7 @@ static void jsonValidFunc( JsonParse x; /* The parse */ int rc = 0; + UNUSED_PARAM(argc); if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 && x.nNode>0 ){ @@ -1427,12 +1407,13 @@ typedef struct JsonEachCursor JsonEachCursor; struct JsonEachCursor { sqlite3_vtab_cursor base; /* Base class - must be first */ u32 iRowid; /* The rowid */ + u32 iBegin; /* The first node of the scan */ u32 i; /* Index in sParse.aNode[] of current row */ u32 iEnd; /* EOF when i equals or exceeds this value */ u8 eType; /* Type of top-level element */ u8 bRecursive; /* True for json_tree(). False for json_each() */ char *zJson; /* Input JSON */ - char *zPath; /* Path by which to filter zJson */ + char *zRoot; /* Path by which to filter zJson */ JsonParse sParse; /* Parse of the input JSON */ }; @@ -1455,16 +1436,17 @@ static int jsonEachConnect( #define JEACH_ID 4 #define JEACH_PARENT 5 #define JEACH_FULLKEY 6 -#define JEACH_JSON 7 -#define JEACH_PATH 8 +#define JEACH_PATH 7 +#define JEACH_JSON 8 +#define JEACH_ROOT 9 UNUSED_PARAM(pzErr); UNUSED_PARAM(argv); UNUSED_PARAM(argc); UNUSED_PARAM(pAux); rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(key,value,type,atom,id,parent,fullkey," - "json HIDDEN,path HIDDEN)"); + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; @@ -1505,14 +1487,14 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ sqlite3_free(p->zJson); - sqlite3_free(p->zPath); + sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); p->iRowid = 0; p->i = 0; p->iEnd = 0; p->eType = 0; p->zJson = 0; - p->zPath = 0; + p->zRoot = 0; } /* Destructor for a jsonEachCursor object */ @@ -1534,16 +1516,10 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; if( p->bRecursive ){ - if( p->i==0 ){ - p->i = 1; - }else{ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->i++; - if( pUp->eType==JSON_OBJECT && (pUp->n + iUp >= p->i) ) p->i++; - } + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; p->iRowid++; - if( p->isParse.nNode ){ + if( p->iiEnd ){ u32 iUp = p->sParse.aUp[p->i]; JsonNode *pUp = &p->sParse.aNode[iUp]; p->eType = pUp->eType; @@ -1597,8 +1573,9 @@ static void jsonEachComputePath( jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); }else{ assert( pUp->eType==JSON_OBJECT ); - if( pNode->eType>=JSON_ARRAY ) pNode--; + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); } } @@ -1629,27 +1606,28 @@ static int jsonEachColumn( break; } case JEACH_VALUE: { - if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; + if( pThis->jnFlags & JNODE_LABEL ) pThis++; jsonReturn(pThis, ctx, 0); break; } case JEACH_TYPE: { - if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; + if( pThis->jnFlags & JNODE_LABEL ) pThis++; sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); break; } case JEACH_ATOM: { - if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; + if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; jsonReturn(pThis, ctx, 0); break; } case JEACH_ID: { - sqlite3_result_int64(ctx, (sqlite3_int64)p->i + (p->eType==JSON_OBJECT)); + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); break; } case JEACH_PARENT: { - if( p->i>0 && p->bRecursive ){ + if( p->i>p->iBegin && p->bRecursive ){ sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); } break; @@ -1660,8 +1638,8 @@ static int jsonEachColumn( if( p->bRecursive ){ jsonEachComputePath(p, &x, p->i); }else{ - if( p->zPath ){ - jsonAppendRaw(&x, p->zPath, (int)strlen(p->zPath)); + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); }else{ jsonAppendChar(&x, '$'); } @@ -1675,18 +1653,20 @@ static int jsonEachColumn( break; } case JEACH_PATH: { - const char *zPath = p->zPath; - if( zPath==0 ){ - if( p->bRecursive ){ - JsonString x; - jsonInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); - break; - } - zPath = "$"; + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; } - sqlite3_result_text(ctx, zPath, -1, SQLITE_STATIC); + /* For json_each() path and root are the same so fall through + ** into the root case */ + } + case JEACH_ROOT: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); break; } default: { @@ -1707,7 +1687,7 @@ static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ /* The query strategy is to look for an equality constraint on the json ** column. Without such a constraint, the table cannot operate. idxNum is -** 1 if the constraint is found, 3 if the constraint and zPath are found, +** 1 if the constraint is found, 3 if the constraint and zRoot are found, ** and 0 otherwise. */ static int jsonEachBestIndex( @@ -1716,7 +1696,7 @@ static int jsonEachBestIndex( ){ int i; int jsonIdx = -1; - int pathIdx = -1; + int rootIdx = -1; const struct sqlite3_index_constraint *pConstraint; UNUSED_PARAM(tab); @@ -1726,7 +1706,7 @@ static int jsonEachBestIndex( if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; switch( pConstraint->iColumn ){ case JEACH_JSON: jsonIdx = i; break; - case JEACH_PATH: pathIdx = i; break; + case JEACH_ROOT: rootIdx = i; break; default: /* no-op */ break; } } @@ -1737,11 +1717,11 @@ static int jsonEachBestIndex( pIdxInfo->estimatedCost = 1.0; pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; - if( pathIdx<0 ){ + if( rootIdx<0 ){ pIdxInfo->idxNum = 1; }else{ - pIdxInfo->aConstraintUsage[pathIdx].argvIndex = 2; - pIdxInfo->aConstraintUsage[pathIdx].omit = 1; + pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[rootIdx].omit = 1; pIdxInfo->idxNum = 3; } } @@ -1756,7 +1736,7 @@ static int jsonEachFilter( ){ JsonEachCursor *p = (JsonEachCursor*)cur; const char *z; - const char *zPath; + const char *zRoot = 0; sqlite3_int64 n; UNUSED_PARAM(idxStr); @@ -1766,11 +1746,11 @@ static int jsonEachFilter( z = (const char*)sqlite3_value_text(argv[0]); if( z==0 ) return SQLITE_OK; if( idxNum&2 ){ - zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ) return SQLITE_OK; - if( zPath[0]!='$' ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zPath); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } } @@ -1794,12 +1774,11 @@ static int jsonEachFilter( JsonNode *pNode; if( idxNum==3 ){ const char *zErr = 0; - p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); - p->zPath = sqlite3_malloc64( n+1 ); - if( p->zPath==0 ) return SQLITE_NOMEM; - memcpy(p->zPath, zPath, (size_t)n+1); - pNode = jsonLookupStep(&p->sParse, 0, p->zPath+1, 0, &zErr); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); if( p->sParse.nErr ){ sqlite3_free(cur->pVtab->zErrMsg); cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); @@ -1811,12 +1790,18 @@ static int jsonEachFilter( }else{ pNode = p->sParse.aNode; } - p->i = (int)(pNode - p->sParse.aNode); + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ pNode->u.iKey = 0; p->iEnd = p->i + pNode->n + 1; - if( !p->bRecursive ) p->i++; + if( p->bRecursive ){ + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } }else{ p->iEnd = p->i+1; } @@ -1901,6 +1886,7 @@ int sqlite3_json_init( int flag; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, { "json_array", -1, 0, jsonArrayFunc }, { "json_array_length", 1, 0, jsonArrayLengthFunc }, { "json_array_length", 2, 0, jsonArrayLengthFunc }, @@ -1918,7 +1904,6 @@ int sqlite3_json_init( /* DEBUG and TESTING functions */ { "json_parse", 1, 0, jsonParseFunc }, { "json_test1", 1, 0, jsonTest1Func }, - { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/main.mk b/main.mk index 7cc09d6c8d..3668c84321 100644 --- a/main.mk +++ b/main.mk @@ -47,6 +47,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth +TCCX += -I$(TOP)/ext/fts5 # Object files for the SQLite library. # @@ -230,6 +231,29 @@ SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h +# FTS5 things +# +FTS5_HDR = \ + $(TOP)/ext/fts5/fts5.h \ + $(TOP)/ext/fts5/fts5Int.h \ + fts5parse.h + +FTS5_SRC = \ + $(TOP)/ext/fts5/fts5_aux.c \ + $(TOP)/ext/fts5/fts5_buffer.c \ + $(TOP)/ext/fts5/fts5_main.c \ + $(TOP)/ext/fts5/fts5_config.c \ + $(TOP)/ext/fts5/fts5_expr.c \ + $(TOP)/ext/fts5/fts5_hash.c \ + $(TOP)/ext/fts5/fts5_index.c \ + fts5parse.c \ + $(TOP)/ext/fts5/fts5_storage.c \ + $(TOP)/ext/fts5/fts5_tokenize.c \ + $(TOP)/ext/fts5/fts5_unicode2.c \ + $(TOP)/ext/fts5/fts5_varint.c \ + $(TOP)/ext/fts5/fts5_vocab.c \ + + # Generated source code files # SRC += \ @@ -558,7 +582,7 @@ parse.h: parse.c parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk cp $(TOP)/src/parse.y . rm -f parse.h - ./lemon $(OPTS) parse.y + ./lemon -s $(OPTS) parse.y mv parse.h parse.h.temp $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h @@ -636,25 +660,6 @@ fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c -# FTS5 things -# -FTS5_SRC = \ - $(TOP)/ext/fts5/fts5.h \ - $(TOP)/ext/fts5/fts5Int.h \ - $(TOP)/ext/fts5/fts5_aux.c \ - $(TOP)/ext/fts5/fts5_buffer.c \ - $(TOP)/ext/fts5/fts5_main.c \ - $(TOP)/ext/fts5/fts5_config.c \ - $(TOP)/ext/fts5/fts5_expr.c \ - $(TOP)/ext/fts5/fts5_hash.c \ - $(TOP)/ext/fts5/fts5_index.c \ - fts5parse.c fts5parse.h \ - $(TOP)/ext/fts5/fts5_storage.c \ - $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c \ - $(TOP)/ext/fts5/fts5_varint.c \ - $(TOP)/ext/fts5/fts5_vocab.c \ - fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h @@ -662,11 +667,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon fts5parse.h: fts5parse.c -fts5.c: $(FTS5_SRC) +fts5.c: $(FTS5_SRC) $(FTS5_HDR) tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . - userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c diff --git a/manifest b/manifest index 90218d1ba1..6edcf0f699 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ C Merge\senhancements\sfrom\strunk. -D 2015-09-03T14:39:33.829 +D 2015-09-15T17:31:47.812 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,63 +104,64 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 -F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h 1950ec0544de667a24c1d8af9b2fde5db7db3bc9 -F ext/fts5/fts5Int.h 45f2ceb3c030f70e2cc4c199e9f700c2f2367f77 -F ext/fts5/fts5_aux.c 044cb176a815f4388308738437f6e130aa384fb0 -F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c fdfa63ae8e527ecfaa50f94063c610429cc887cf -F ext/fts5/fts5_expr.c d075d36c84975a1cfcf070442d28e28027b61c25 +F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 +F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc +F ext/fts5/fts5Int.h 666aba8432940a8449a3bd4636e898fe906ed95d +F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e +F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 +F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 +F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c fc47ad734dfb55765b7542a345cee981170e7caa -F ext/fts5/fts5_storage.c 22ec9b5d35a39e2b5b65daf4ba7cd47fbb2d0df5 -F ext/fts5/fts5_tcl.c 96a3b9e982c4a64a242eefd752fa6669cd405a67 -F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 -F ext/fts5/fts5_tokenize.c 2836f6728bd74c7efac7487f5d9c27ca3e1b509c +F ext/fts5/fts5_index.c 4fdbc0a321e3a1d73741a623d7aea4db78d6a86d +F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c +F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 +F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 +F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf +F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7 -F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0 +F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc +F ext/fts5/test/fts5aa.test 4804f237005bb4ba8ea4a76120d8011ebcb5d611 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c -F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d +F ext/fts5/test/fts5ad.test e3dfb150fce971b4fd832498c29f56924d451b63 F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 -F ext/fts5/test/fts5ah.test b9e78fa986a7bd564ebadfb244de02c84d7ac3ae +F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 -F ext/fts5/test/fts5al.test 440d77c0b39ba73bad2ceb8986c2bb1093570735 +F ext/fts5/test/fts5al.test 5c79525671862861906fa0a848da462a8473eafb F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb -F ext/fts5/test/fts5columnsize.test 97dc6bd66c91009d00407aa078dd5e9e8eb22f99 +F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 -F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6 -F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b -F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3 -F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1 +F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 +F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c +F ext/fts5/test/fts5corrupt3.test a1429635c30f922079189e35b2b631eb7463a2fb +F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b -F ext/fts5/test/fts5ea.test 451bb37310ee6df8ef72e4354fda5621b3b51448 -F ext/fts5/test/fts5eb.test 46f49497edc25ef3b2bff9fb6d75b6d201e2b39e +F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 +F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43 F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215 +F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423 +F ext/fts5/test/fts5fault7.test f62ed4d98f137eb03f1db94d1fa41b17a771d971 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 -F ext/fts5/test/fts5matchinfo.test ee6e7b130096c708c12049fa9c1ceb628954c4f9 +F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 @@ -171,17 +172,19 @@ F ext/fts5/test/fts5prefix.test 552a462f0e8595676611f41643de217fb4ac2808 F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 -F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 +F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 +F ext/fts5/test/fts5simple.test f629e24a35a9f31cfb16c9920e8c2316e3d93e94 +F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 -F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8 +F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c -F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556 +F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51 -F ext/fts5/tool/showfts5.tcl fb62e8eae6d862afdd22f367e286fb886d5e1ab6 +F ext/fts5/tool/showfts5.tcl 9eaf6c3df352f98a2ab5ce1921dd94128ab1381d F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -192,7 +195,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c bd51e8c1e8ce580e6f21493bd8e94ed5aca3d777 +F ext/misc/json1.c f35d00fbd79a7e23af18d7630a2fcf22dce3692b F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -258,7 +261,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 8da13ed011a7ae19450b7554910ff4afa3bd22b7 +F main.mk 58eb74de702467c3b71cdf06f213cefe7f5ce544 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -273,37 +276,37 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 4911e1f18fc11b60edbc6410643e938762969a6a -F src/analyze.c e4ad1495090d6b7bf58b927e1267fc211ad5e7da +F src/analyze.c 4c308880cf53c558070cb8513bdff4ffb1a38a77 F src/attach.c e944d0052b577703b9b83aac1638452ff42a8395 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 3d759d9943ee2ce422a2bb1f7810e7447a907e97 -F src/btree.h 6310645beddbd3eef89fa9b6d1062065976a93c8 +F src/btree.c 9d19d3f8dcddd1b644b72f1c6613d44692a7a846 +F src/btree.h 9cc758f0f1e0002049d9eb0232f9783f21f25ac5 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 6c3a8a9b21402a6be98126f7d86b76527e68ca67 +F src/build.c 8a86f4203ac8a9ac0734f242a96f043edffb6018 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b -F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 -F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a -F src/delete.c 5b4835982c31f12b256dc4d814363a11e7ef34cf -F src/expr.c 3783aa19150be01a2ae92e1eb429c84a2313c8fc +F src/date.c fb1c99172017dcc8e237339132c91a21a0788584 +F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7 +F src/delete.c 5ab483f15aaf202290227d5dd1f10a7ecce3d6b1 +F src/expr.c 572ebfeb5c848e8d32096c823f87baa1eb988391 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f -F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f +F src/func.c ecdd69ec6a1e406f04cc73324be2ebbf6354197f F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c bcff4a416177ed90faa8dba65f0266ddad2aeb76 +F src/insert.c 9748a37e058256eb2ead69f028ab85ebf203ad15 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e -F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 -F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 +F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012 +F src/loadext.c f0b66d28e377fd6c6d36cc9d92df1ff251ebee44 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 -F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d +F src/malloc.c 3a37ce6979a40f499d8cea9e9ab4e8517854d35d F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -311,11 +314,11 @@ F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 -F src/mutex.c 529e95739f815300a33c73fd8a7d6bdf0c24bd18 +F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c b0d280089df0f49545f1318f45d61d07d2f674a8 -F src/mutex_w32.c b601f9e3073f7bd2c1f42a8c0ce59e42d6a08f85 +F src/mutex_unix.c 8cfa6e83c618d2fcae0fe63f4d2b5bb16b11a97a +F src/mutex_w32.c 2e025e6642eaf27597403690980f560d1a91f62c F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf @@ -324,25 +327,25 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 76f493ed71c4154338049dee1bf6e47f69c74a55 F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f +F src/pager.c 4784012f80b2197c61ff6eaf4f5c7026d93253fd F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 -F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 +F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c b31af9dbc83b9c68e87d681b8453a9605f28e734 +F src/pcache1.c a0c0bb29f7bd720743a16a95eb5dedba3ade15bc F src/pragma.c d71b813e67bf03f3116b9dd5164fbfd81ec673a2 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 0c4bcdd1c2e2521024f0a69cb5eb334f86b3652a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c f2ef256786a6435efddd64a632fea89c8be62215 +F src/resolve.c 3126f7694b8ce0f97282d7dd3a5198b8fa18dce9 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c bb99c37242673405683992085a0fcff380b70575 -F src/shell.c bbe2bab590b7dd04dd8f9119d4473cb8c52906e3 -F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 +F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 +F src/sqlite.h.in 50f83145c6543000b7d27525ecaec59a23d8280b F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h c452759fafe7d6ed9bbdfec19d327a2b77e85d2e +F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 +F src/sqliteInt.h b108aac991a28e4208f526df54f82995316c56ee F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -365,7 +368,7 @@ F src/test_config.c fb2e5d354d9a077f5fbb261652eff4787deb104f F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c f1ac201465472e76a73e2f3695c3553c63e7322a +F src/test_func.c 0d9c25956152adefee8881c6fadc8354793764d0 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 @@ -393,31 +396,31 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 -F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 -F src/treeview.c 46036cbbceada0836833531b2d963edbca3d9cfa +F src/tokenize.c 83c6ed569423a3af83a83973b444cf7123be33a6 +F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f -F src/update.c 795fba8ebadeb194285b0a73a07f7ad5ae4d0410 +F src/update.c eb7ab3ff2928628692a4f14be397c95f4a681d97 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c 9d85fa72a73f7f07789d76602f724be63e998ddc +F src/vdbe.c 1704a79d60f9c316adac0cbce09cae1ffc4b6af8 F src/vdbe.h efb7a8c1459e31f3ea4377824c6a7e4cb5068637 -F src/vdbeInt.h 7258d75fc2dad0bccdef14d7d8d2fd50fd1bf2d2 -F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f +F src/vdbeInt.h c1508eb4ee04d4db7bdfca5620cfa8ad632fdd62 +F src/vdbeapi.c 0d890f57caf143b114a95ce699e59af51359c508 F src/vdbeaux.c 7dccd114c9ee5f801709daecee36464fe6ce71af -F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 -F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 +F src/vdbeblob.c 1d7b97115e7bbac4c318db416d2ca83fc779544a +F src/vdbemem.c 19b3036aa4d676e7103b0fb5efd6327da455f915 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2ecfe020c10e0a0c7b078203fdba2fae844744bc F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 +F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 6055bee2f4aff2934ea1965c25c4d1f6368dc0eb -F src/whereInt.h 901c17c1e3c82745ad9b85b4471543fa59c980e9 -F src/wherecode.c 93fa794fcd1aba23454cc15bca2b617bca21ec0a -F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b +F src/where.c c03895cdb2624a07e75c95c6889a32a15929a27a +F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647 +F src/wherecode.c c80f4c7866f78b298b81fdda60006d31125b6f68 +F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -567,9 +570,10 @@ F test/date.test 42973251b9429f2c41b77eb98a7b0b0ba2d3b2c0 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d -F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 +F test/delete.test e1bcdf8926234e27aac24b346ad83d3329ec8b6f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab +F test/delete4.test d9e7d553a939597b27d205b022d769469f361c1f F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 @@ -581,20 +585,20 @@ F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 -F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 -F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a +F test/e_createtable.test d4c6059d44dcd4b636de9aae322766062b471844 +F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 8f5fdd7261e2d746813b0c6a1c0e34824ad3c5ad +F test/e_expr.test 03a84a6fa9bd3472112d6bd4599f5269f5f74803 F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 -F test/e_insert.test 0e63edc037afe738bb81a626a676811ed7862c90 -F test/e_reindex.test 57d439f6c644befc8274ac93cf2f5449cf2736c1 -F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 +F test/e_insert.test 3de217e95094d3d165992a6de1164bbc4bd92dc7 +F test/e_reindex.test 2bebf7b393e519198b7c654407221cf171a439b8 +F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 -F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 +F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 @@ -622,7 +626,7 @@ F test/fkey1.test de5b287f6a480b36bd51e8debcf48168e26e4ed2 F test/fkey2.test f3d27ecba480a348c328965d154214719bb158a9 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d -F test/fkey5.test 56bcb5a6e8b725b17febc267fb041a6695e86853 +F test/fkey5.test 5a373303f201ac03c22ba1ef17a733d3f56e611a F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 F test/fkey8.test 8f08203458321e6c19a263829de4cfc936274ab0 @@ -773,14 +777,15 @@ F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test 4d990005a67a36984e4f1a5f1bdccea8d08da4ee -F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 -F test/index3.test b6ec456cf3b81d9a32123fe7e449bde434db338b +F test/index.test fe3c7a1aad82af92623747e9c3f3aa94ccd51238 +F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 +F test/index3.test fa3e49bbaa4f38091c9c742e36a1abe67c4ef1fc F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c -F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 +F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 +F test/indexexpr1.test 4feec154aadacb033b41acc1760a18edc4c60470 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -811,7 +816,8 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 11535d8986184500f4c30cc2f0b154b4ab05cc4e +F test/json101.test e20d2421c531db32fad59c5e06e80af0b1b002c8 +F test/json102.test 4e1403cb06481ab160cf471c3c139820498e0563 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -917,8 +923,8 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test ac3b00c299250cc087d4a527b5c75a0f8aef4e54 -F test/pragma.test be7195f0aa72bdb8a512133e9640ac40f15b57a2 -F test/pragma2.test 8e72df3a16c0fda748ad52abf79cb8256b04a6fe +F test/pragma.test a44253f911e7d50127d4a08f927f47c861a4c772 +F test/pragma2.test b5e2ce4c892afceb308c6ae6163a9099b2a0d8d7 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 @@ -936,14 +942,14 @@ F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl cd2de2749aab7f45b2fe91b4a05431fc08e1692a +F test/releasetest.tcl 5af0ca3d6a12471ffd9993c3efc00a553dfb2d41 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 F test/rollbackfault.test 6a004f71087cc399296cffbb5429ea6da655ae65 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 +F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d @@ -1032,12 +1038,13 @@ F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a +F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/tabfunc01.test a12eba3f48a03a6626f985734ecc28132381fa9b -F test/table.test 33bf0d1fd07f304582695184b8e6feb017303816 +F test/tabfunc01.test fa9d8dfc75747019e0be98d3b6ac68d18632d328 +F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 7fb866443c7deceed22b63948ccd6f76b52ad054 @@ -1287,10 +1294,10 @@ F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 1ff3d9f8da0a6c0dc5ccfd38d9225b2cdb5b6afb +F test/where.test 66d4c107e82dfe86c01a96277b77e7a8809aff0b F test/where2.test af78c55589cbc82d793449493adba0dc3d659f23 F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e -F test/where4.test 68aa5ad796e33816db2078bc0f6de719c7a0e21f +F test/where4.test 44f506bf1737cf0fa4fc795e340208250f1fcd89 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 @@ -1336,8 +1343,8 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c b9109f59b57e7b6f101c4fe644c8361ba6dee969 -F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc +F tool/lemon.c 039f813b520b9395740c52f9cbf36c90b5d8df03 +F tool/lempar.c 3617143ddb9b176c3605defe6a9c798793280120 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f @@ -1363,7 +1370,7 @@ F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b -F tool/spaceanal.tcl 63a415385a66fdbf736bfd204a31c6d851ed8da6 +F tool/spaceanal.tcl 93c1fdc9733c525b17a2024c7df193daa002e037 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1381,7 +1388,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29570a604806e2a60a8eef7eaf1dce022d2a7004 847387ec8e6fef283899578fb232b2c23b00ee5b -R af53d4591486f700f0b5e9bcc7196ecf +P 25ee3000e94d60d8c1d7b980f416dcc33eb11105 1d018c35b9e81982df036f5e62a4a42219b54e02 +R 094d10cbcbe8f2ec763e09bd3b0c23d3 U drh -Z 6458a4b1081b59ccab00b014bcb486e3 +Z 515bf273b300e1c6b0d004e464776ff6 diff --git a/manifest.uuid b/manifest.uuid index 73014e7f12..3cab444f7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25ee3000e94d60d8c1d7b980f416dcc33eb11105 \ No newline at end of file +66fe06832614010d3156d7b21a760af9957018cc \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 69d4b659cd..ad752d2c0e 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1186,6 +1186,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -1235,12 +1236,10 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, - pIdx->aiColumn[0], regSample); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, i, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif diff --git a/src/btree.c b/src/btree.c index de6b089a31..b8aef7bd5c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -591,6 +591,49 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } +/* +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. +** +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. +*/ +static int saveCursorKey(BtCursor *pCur){ + int rc; + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); + assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ + + /* If this is an intKey table, then the above call to BtreeKeySize() + ** stores the integer key in pCur->nKey. In this case this value is + ** all that is required. Otherwise, if pCur is not open on an intKey + ** table, then malloc space for and store the pCur->nKey bytes of key + ** data. */ + if( 0==pCur->curIntKey ){ + void *pKey = sqlite3Malloc( pCur->nKey ); + if( pKey ){ + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM; + } + } + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} /* ** Save the current cursor position in the variables BtCursor.nKey @@ -611,30 +654,8 @@ static int saveCursorPosition(BtCursor *pCur){ }else{ pCur->skipNext = 0; } - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ - if( 0==pCur->curIntKey ){ - void *pKey = sqlite3Malloc( pCur->nKey ); - if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); - if( rc==SQLITE_OK ){ - pCur->pKey = pKey; - }else{ - sqlite3_free(pKey); - } - }else{ - rc = SQLITE_NOMEM; - } - } - assert( !pCur->curIntKey || !pCur->pKey ); + rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; @@ -8047,10 +8068,15 @@ end_insert: } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at an arbitrary location. +** Delete the entry that the cursor is pointing to. +** +** If the second parameter is zero, then the cursor is left pointing at an +** arbitrary location after the delete. If it is non-zero, then the cursor +** is left in a state such that the next call to BtreeNext() or BtreePrev() +** moves it to the same row as it would if the call to BtreeDelete() had +** been omitted. */ -int sqlite3BtreeDelete(BtCursor *pCur){ +int sqlite3BtreeDelete(BtCursor *pCur, int bPreserve){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -8059,6 +8085,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ u16 szCell; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -8088,10 +8115,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ + ** making any modifications. */ if( pCur->curFlags & BTCF_Multiple ){ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; @@ -8103,6 +8127,31 @@ int sqlite3BtreeDelete(BtCursor *pCur){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell, &szCell); @@ -8156,7 +8205,22 @@ int sqlite3BtreeDelete(BtCursor *pCur){ } if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( bSkipnext ){ + assert( bPreserve && pCur->iPage==iCellDepth ); + assert( pPage->nCell>0 && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->aiIdx[iCellDepth] = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + pCur->eState = CURSOR_REQUIRESEEK; + } + } } return rc; } diff --git a/src/btree.h b/src/btree.h index e0c9ab0ab1..910abe6b85 100644 --- a/src/btree.h +++ b/src/btree.h @@ -222,7 +222,7 @@ int sqlite3BtreeMovetoUnpacked( ); int sqlite3BtreeCursorHasMoved(BtCursor*); int sqlite3BtreeCursorRestore(BtCursor*, int*); -int sqlite3BtreeDelete(BtCursor*); +int sqlite3BtreeDelete(BtCursor*, int); int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, int nZero, int bias, int seekResult); diff --git a/src/build.c b/src/build.c index a5e838ccb9..eee62ee151 100644 --- a/src/build.c +++ b/src/build.c @@ -357,12 +357,14 @@ Table *sqlite3LocateTable( if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; #ifndef SQLITE_OMIT_VIRTUALTABLE - /* If zName is the not the name of a table in the schema created using - ** CREATE, then check to see if it is the name of an virtual table that - ** can be an eponymous virtual table. */ - Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); - if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ - return pMod->pEpoTab; + if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } } #endif if( zDbase ){ @@ -372,7 +374,7 @@ Table *sqlite3LocateTable( } pParse->checkSchema = 1; } -#if SQLITE_USER_AUTHENICATION +#if SQLITE_USER_AUTHENTICATION else if( pParse->db->auth.authLevelpPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -983,6 +986,8 @@ void sqlite3StartTable( int j1; int fileFormat; int reg1, reg2, reg3; + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -1027,7 +1032,7 @@ void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); @@ -1309,7 +1314,8 @@ void sqlite3AddPrimaryKey( nTerm = pList->nExpr; for(i=0; ia[i].pExpr); - if( pCExpr && pCExpr->op==TK_ID ){ + assert( pCExpr!=0 ); + if( pCExpr->op==TK_ID ){ const char *zCName = pCExpr->u.zToken; for(iCol=0; iColnCol; iCol++){ if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ @@ -2848,6 +2854,30 @@ Index *sqlite3AllocateIndexObject( return p; } +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the epxression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will @@ -2889,7 +2919,6 @@ Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ @@ -3061,7 +3090,8 @@ Index *sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr && pExpr->op==TK_COLLATE ){ + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -3102,39 +3132,52 @@ Index *sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store -2 in aiColumn[]. ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName; - Expr *pCExpr; - int requestedSortOrder; + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ char *zColl; /* Collation sequence name */ + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); - if( pCExpr->op!=TK_ID ){ - sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported"); - continue; + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); + pIndex->aColExpr = pCopy; + if( !db->mallocFailed ){ + assert( pCopy!=0 ); + pListItem = &pCopy->a[i]; + } + } + j = -2; + pIndex->aiColumn[i] = -2; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + pIndex->aiColumn[i] = (i16)j; } - zColName = pCExpr->u.zToken; - for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; - } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; - } - assert( j<=0x7fff ); - pIndex->aiColumn[i] = (i16)j; + zColl = 0; if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; zColl = pListItem->pExpr->u.zToken; @@ -3144,21 +3187,26 @@ Index *sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else{ + }else if( j>=0 ){ zColl = pTab->aCol[j].zColl; - if( !zColl ) zColl = "BINARY"; } + if( !zColl ) zColl = "BINARY"; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; - if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ if( pPk ){ for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; + assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -3209,6 +3257,7 @@ Index *sqlite3CreateIndex( for(k=0; knKeyCol; k++){ const char *z1; const char *z2; + assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; @@ -3240,6 +3289,7 @@ Index *sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ + assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); @@ -3269,7 +3319,7 @@ Index *sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + else if( HasRowid(pTab) || pTblName!=0 ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -4099,12 +4149,16 @@ void sqlite3UniqueConstraint( Table *pTab = pIdx->pTable; sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); - for(j=0; jnKeyCol; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3StrAccumAppendAll(&errMsg, pTab->zName); - sqlite3StrAccumAppend(&errMsg, ".", 1); - sqlite3StrAccumAppendAll(&errMsg, zCol); + if( pIdx->aColExpr ){ + sqlite3XPrintf(&errMsg, 0, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3XPrintf(&errMsg, 0, "%s.%s", pTab->zName, zCol); + } } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, diff --git a/src/date.c b/src/date.c index 6b11d9904a..8a66eae900 100644 --- a/src/date.c +++ b/src/date.c @@ -1115,14 +1115,14 @@ static void currentTimeFunc( void sqlite3RegisterDateTimeFunctions(void){ static SQLITE_WSD FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - FUNCTION(julianday, -1, 0, 0, juliandayFunc ), - FUNCTION(date, -1, 0, 0, dateFunc ), - FUNCTION(time, -1, 0, 0, timeFunc ), - FUNCTION(datetime, -1, 0, 0, datetimeFunc ), - FUNCTION(strftime, -1, 0, 0, strftimeFunc ), - FUNCTION(current_time, 0, 0, 0, ctimeFunc ), - FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - FUNCTION(current_date, 0, 0, 0, cdateFunc ), + DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), + DFUNCTION(date, -1, 0, 0, dateFunc ), + DFUNCTION(time, -1, 0, 0, timeFunc ), + DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), + DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), diff --git a/src/dbstat.c b/src/dbstat.c index c36be020af..f43b14881f 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -16,6 +16,9 @@ ** information from an SQLite database in order to implement the ** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script ** for an example implementation. +** +** Additional information is available on the "dbstat.html" page of the +** official SQLite documentation. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -64,7 +67,8 @@ " unused INTEGER, /* Bytes of unused space on this page */" \ " mx_payload INTEGER, /* Largest payload size of all cells */" \ " pgoffset INTEGER, /* Offset of page in file */" \ - " pgsize INTEGER /* Size of the page */" \ + " pgsize INTEGER, /* Size of the page */" \ + " schema TEXT HIDDEN /* Database schema being analyzed */" \ ");" @@ -102,6 +106,7 @@ struct StatCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Iterates through set of root pages */ int isEof; /* After pStmt has returned SQLITE_DONE */ + int iDb; /* Schema used for this query */ StatPage aPage[32]; int iPage; /* Current entry in aPage[] */ @@ -179,9 +184,32 @@ static int statDisconnect(sqlite3_vtab *pVtab){ /* ** There is no "best-index". This virtual table always does a linear -** scan of the binary VFS log file. +** scan. However, a schema=? constraint should cause this table to +** operate on a different database schema, so check for it. +** +** idxNum is normally 0, but will be 1 if a schema=? constraint exists. */ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + + pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + + /* Look for a valid schema=? constraint. If found, change the idxNum to + ** 1 and request the value of that constraint be sent to xFilter. And + ** lower the cost estimate to encourage the constrained version to be + ** used. + */ + for(i=0; inConstraint; i++){ + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pIdxInfo->aConstraint[i].iColumn!=10 ) continue; + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + /* Records are always returned in ascending order of (name, path). ** If this will satisfy the client, set the orderByConsumed flag so that @@ -201,7 +229,6 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = 10.0; return SQLITE_OK; } @@ -211,36 +238,18 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ StatTable *pTab = (StatTable *)pVTab; StatCursor *pCsr; - int rc; pCsr = (StatCursor *)sqlite3_malloc64(sizeof(StatCursor)); if( pCsr==0 ){ - rc = SQLITE_NOMEM; + return SQLITE_NOMEM; }else{ - char *zSql; memset(pCsr, 0, sizeof(StatCursor)); pCsr->base.pVtab = pVTab; - - zSql = sqlite3_mprintf( - "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" - " UNION ALL " - "SELECT name, rootpage, type" - " FROM \"%w\".sqlite_master WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pTab->iDb].zName); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ){ - sqlite3_free(pCsr); - pCsr = 0; - } + pCsr->iDb = pTab->iDb; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; - return rc; + return SQLITE_OK; } static void statClearPage(StatPage *p){ @@ -265,6 +274,7 @@ static void statResetCsr(StatCursor *pCsr){ pCsr->iPage = 0; sqlite3_free(pCsr->zPath); pCsr->zPath = 0; + pCsr->isEof = 0; } /* @@ -427,7 +437,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ char *z; StatCursor *pCsr = (StatCursor *)pCursor; StatTable *pTab = (StatTable *)pCursor->pVtab; - Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + Btree *pBt = pTab->db->aDb[pCsr->iDb].pBt; Pager *pPager = sqlite3BtreePager(pBt); sqlite3_free(pCsr->zPath); @@ -565,9 +575,43 @@ static int statFilter( int argc, sqlite3_value **argv ){ StatCursor *pCsr = (StatCursor *)pCursor; + StatTable *pTab = (StatTable*)(pCursor->pVtab); + char *zSql; + int rc = SQLITE_OK; + char *zMaster; + if( idxNum==1 ){ + const char *zDbase = (const char*)sqlite3_value_text(argv[0]); + pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase); + if( pCsr->iDb<0 ){ + sqlite3_free(pCursor->pVtab->zErrMsg); + pCursor->pVtab->zErrMsg = sqlite3_mprintf("no such schema: %s", zDbase); + return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + }else{ + pCsr->iDb = pTab->iDb; + } statResetCsr(pCsr); - return statNext(pCursor); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + zMaster = pCsr->iDb==1 ? "sqlite_temp_master" : "sqlite_master"; + zSql = sqlite3_mprintf( + "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" + " UNION ALL " + "SELECT name, rootpage, type" + " FROM \"%w\".%s WHERE rootpage!=0" + " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster); + if( zSql==0 ){ + return SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + rc = statNext(pCursor); + } + return rc; } static int statColumn( @@ -604,10 +648,15 @@ static int statColumn( case 8: /* pgoffset */ sqlite3_result_int64(ctx, pCsr->iOffset); break; - default: /* pgsize */ - assert( i==9 ); + case 9: /* pgsize */ sqlite3_result_int(ctx, pCsr->szPage); break; + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + int iDb = pCsr->iDb; + sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC); + break; + } } return SQLITE_OK; } diff --git a/src/delete.c b/src/delete.c index 539abb1625..860b2cf115 100644 --- a/src/delete.c +++ b/src/delete.c @@ -235,7 +235,7 @@ void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -247,12 +247,12 @@ void sqlite3DeleteFrom( int iRowSet = 0; /* Register for rowset of rows to delete */ int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ - int addrDelete = 0; /* Jump directly to the delete logic */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ + int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -276,9 +276,11 @@ void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 +# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -359,8 +361,10 @@ void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -375,6 +379,8 @@ void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -395,13 +401,17 @@ void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, - iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass==ONEPASS_OFF ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -411,6 +421,7 @@ void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=(-1) ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -421,11 +432,10 @@ void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( okOnePass ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. - */ + ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ aToOpen = sqlite3DbMallocRaw(db, nIdx+2); if( aToOpen==0 ){ @@ -437,27 +447,27 @@ void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); - addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ - }else if( pPk ){ - /* Construct a composite key for the row to be deleted and remember it */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(pParse->db, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - /* Get the rowid of the row to be deleted and remember it in the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } } - /* End of the WHERE loop */ - sqlite3WhereEnd(pWInfo); - if( okOnePass ){ - /* Bypass the delete logic below if the WHERE loop found zero rows */ + /* If this DELETE cannot use the ONEPASS strategy, this is the + ** end of the WHERE loop */ + if( eOnePass!=ONEPASS_OFF ){ addrBypass = sqlite3VdbeMakeLabel(v); - sqlite3VdbeGoto(v, addrBypass); - sqlite3VdbeJumpHere(v, addrDelete); + }else{ + sqlite3WhereEnd(pWInfo); } /* Unless this is a view, open cursors for the table we are @@ -466,20 +476,23 @@ void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( okOnePass ){ - /* Just one row. Hence the top-of-loop is a no-op */ + if( eOnePass!=ONEPASS_OFF ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); @@ -507,13 +520,18 @@ void sqlite3DeleteFrom( #endif { int count = (pParse->nested==0); /* True to count changes */ + int iIdxNoSeek = -1; + if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ + iIdxNoSeek = aiCurOnePass[1]; + } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, okOnePass); + iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } /* End of the loop over all rowids/primary-keys. */ - if( okOnePass ){ + if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); @@ -585,6 +603,25 @@ delete_from_cleanup: ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0), then its position should +** be preserved following the delete operation. Or, if iIdxNoSeek is not +** a valid cursor number, the position of iDataCur should be preserved +** instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an +** index cursor (from within array of cursors starting at iIdxCur) that +** already points to the index entry to be deleted. */ void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -596,7 +633,8 @@ void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 bNoSeek /* iDataCur is already pointing to the row to delete */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -613,7 +651,7 @@ void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ){ + if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -673,11 +711,15 @@ void sqlite3GenerateRowDelete( ** a view (in which case the only effect of the DELETE statement is to ** fire the INSTEAD OF triggers). */ if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); if( count ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } + if( iIdxNoSeek>=0 ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + sqlite3VdbeChangeP5(v, eMode==ONEPASS_MULTI); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -720,7 +762,8 @@ void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -736,11 +779,12 @@ void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -789,14 +833,13 @@ int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; - Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iPartIdxTab = iDataCur; + pParse->iSelfTab = iDataCur; sqlite3ExprCachePush(pParse); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); @@ -808,9 +851,14 @@ int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], - regBase+j); + if( pPrior + && pPrior->aiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]>=(-1) + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. diff --git a/src/expr.c b/src/expr.c index c7ebf93641..aa9cc7b528 100644 --- a/src/expr.c +++ b/src/expr.c @@ -91,7 +91,7 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ } /* -** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** Skip over any TK_COLLATE operators and any unlikely() ** or likelihood() function at the root of an expression. */ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -102,7 +102,7 @@ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; } } @@ -2448,6 +2448,28 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ } } +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol>=(-1) ){ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + return; + } + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); +} + /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -2633,8 +2655,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; break; }else{ - /* Deleting from a partial index */ - iTab = pParse->iPartIdxTab; + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab; } } inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, @@ -2694,10 +2717,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iTable; break; } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; - } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ @@ -3781,7 +3800,9 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ return 2; } if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ - if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op==TK_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } diff --git a/src/func.c b/src/func.c index 6ecd743969..8ea1169327 100644 --- a/src/func.c +++ b/src/func.c @@ -1737,15 +1737,15 @@ void sqlite3RegisterGlobalFunctions(void){ VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), #if SQLITE_USER_AUTHENTICATION FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), #endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), @@ -1757,8 +1757,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(soundex, 1, 0, 0, soundexFunc ), #endif #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), #endif AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), diff --git a/src/insert.c b/src/insert.c index 93d1f785d8..53b429c1f4 100644 --- a/src/insert.c +++ b/src/insert.c @@ -88,7 +88,18 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; - pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + if( x>=0 ){ + pIdx->zColAff[n] = pTab->aCol[x].affinity; + }else if( x==(-1) ){ + pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + }else{ + char aff; + assert( x==(-2) ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff==0 ) aff = SQLITE_AFF_BLOB; + pIdx->zColAff[n] = aff; + } } pIdx->zColAff[n] = 0; } @@ -1336,10 +1347,13 @@ void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1); - }else if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + regNewData, 1, 0, OE_Replace, + ONEPASS_SINGLE, -1); + }else{ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } } seenReplace = 1; break; @@ -1394,15 +1408,22 @@ void sqlite3GenerateConstraintChecks( for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField<0 || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + if( iField==(-2) ){ + pParse->ckBase = regNewData+1; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->ckBase = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - x = iField + regNewData + 1; + if( iField==(-1) || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -1510,7 +1531,8 @@ void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, pIdx==pPk); + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); seenReplace = 1; break; } @@ -1723,6 +1745,13 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } + if( pSrc->aiColumn[i]==(-2) ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } diff --git a/src/lempar.c b/src/lempar.c index b6c60a25b6..5e5a11aeaa 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -56,15 +56,19 @@ ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ %% -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -91,16 +95,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -159,9 +167,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -395,10 +407,10 @@ static int yy_find_shift_action( int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } + if( stateno>=YY_MIN_REDUCE ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ @@ -499,7 +511,29 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ } /* -** Perform a shift action. +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + +/* +** Perform a shift action. Return the number of errors. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ @@ -532,16 +566,7 @@ static void yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - } -#endif + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -574,8 +599,9 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -613,9 +639,9 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). ** That gives a significant speed improvement. */ @@ -625,13 +651,12 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; - }else -#endif - { + yyTraceShift(yypParser, yyact); + }else{ yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); yy_accept(yypParser); } } @@ -755,12 +780,13 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; yy_shift(yypParser,yyact,yymajor,&yyminorunion); yypParser->yyerrcnt--; yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); #ifdef YYERRORSYMBOL @@ -810,7 +836,7 @@ void Parse( yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -860,5 +886,10 @@ void Parse( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + } +#endif return; } diff --git a/src/loadext.c b/src/loadext.c index 1d398c54ce..b4b981e548 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -407,7 +407,10 @@ static const sqlite3_api_routines sqlite3Apis = { (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, sqlite3_value_free, sqlite3_result_zeroblob64, - sqlite3_bind_zeroblob64 + sqlite3_bind_zeroblob64, + /* Version 3.8.12 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype }; /* diff --git a/src/malloc.c b/src/malloc.c index 23d7598ae3..f20eb6e796 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -45,7 +45,7 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - sqlite3_int64 alarmThreshold; /* The soft heap limit */ + sqlite3_int64 alarmThreshold; /* The soft heap limit */ /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -73,59 +73,20 @@ sqlite3_mutex *sqlite3MallocMutex(void){ return mem0.mutex; } -/* -** Return the amount of memory currently in use. -*/ -static sqlite3_int64 memInUse(void){ - assert( sqlite3_mutex_held(mem0.mutex) ); - return sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); -} - -/* -** Called when the soft heap limit is exceeded for an allocation -** of nBytes. -*/ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -static void sqlite3HeapLimitExceeded(int nByte){ - sqlite3_int64 excess = memInUse() + nByte - mem0.alarmThreshold; - sqlite3_mutex_leave(mem0.mutex); - sqlite3_release_memory((int)(excess & 0x7fffffff)); - sqlite3_mutex_enter(mem0.mutex); -} -#else -# define sqlite3HeapLimitExceeded(X) /* no-op */ -#endif - -/* -** Check to see if increasing the total memory usage by nNew bytes -** will exceed the soft heap limit. -** -** If the soft heap limit is exceeded, set the mem0.nearlyFull flag -** and invoke sqlite3HeapLimitExceeded() to try to free up some -** memory. -*/ -static void sqlite3CheckSoftHeapLimit(int nNew){ - assert( sqlite3_mutex_held(mem0.mutex) ); - if( mem0.alarmThreshold>0 ){ - if( mem0.alarmThreshold-nNew >= memInUse() ){ - mem0.nearlyFull = 1; - sqlite3HeapLimitExceeded(nNew); - }else{ - mem0.nearlyFull = 0; - } - } -} - #ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. First deprecated 2007-11-05. Changed -** into a no-op on 2015-09-02. +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. */ int sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ + (void)xCallback; + (void)pArg; + (void)iThreshold; return SQLITE_OK; } #endif @@ -136,20 +97,24 @@ int sqlite3_memory_alarm( */ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; + sqlite3_int64 excess; + sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - if( n>0 ){ - mem0.alarmThreshold = n; - sqlite3CheckSoftHeapLimit(0); - }else if( n==0 ){ - mem0.alarmThreshold = 0; - mem0.nearlyFull = 0; + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; } void sqlite3_soft_heap_limit(int n){ @@ -240,6 +205,16 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ return mx; } +/* +** Trigger the alarm +*/ +static void sqlite3MallocAlarm(int nByte){ + if( mem0.alarmThreshold<=0 ) return; + sqlite3_mutex_leave(mem0.mutex); + sqlite3_release_memory(nByte); + sqlite3_mutex_enter(mem0.mutex); +} + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -250,11 +225,19 @@ static int mallocWithAlarm(int n, void **pp){ assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); - sqlite3CheckSoftHeapLimit(nFull); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.alarmThreshold - nFull ){ + mem0.nearlyFull = 1; + sqlite3MallocAlarm(nFull); + }else{ + mem0.nearlyFull = 0; + } + } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmThreshold ){ - sqlite3HeapLimitExceeded(nFull); + if( p==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } #endif @@ -537,14 +520,15 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; - sqlite3CheckSoftHeapLimit(nDiff); + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); + } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( pNew==0 && mem0.alarmThreshold ){ - sqlite3HeapLimitExceeded((int)nBytes); + if( pNew==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } -#endif if( pNew ){ nNew = sqlite3MallocSize(pNew); sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); diff --git a/src/mutex.c b/src/mutex.c index 64efd3b05e..6f1bc9767d 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -22,7 +22,7 @@ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ #ifndef SQLITE_MUTEX_OMIT @@ -53,8 +53,10 @@ int sqlite3MutexInit(void){ pTo->xMutexLeave = pFrom->xMutexLeave; pTo->xMutexHeld = pFrom->xMutexHeld; pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -89,6 +91,7 @@ sqlite3_mutex *sqlite3_mutex_alloc(int id){ if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -97,6 +100,7 @@ sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -105,6 +109,7 @@ sqlite3_mutex *sqlite3MutexAlloc(int id){ */ void sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -115,6 +120,7 @@ void sqlite3_mutex_free(sqlite3_mutex *p){ */ void sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -126,6 +132,7 @@ void sqlite3_mutex_enter(sqlite3_mutex *p){ int sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -139,6 +146,7 @@ int sqlite3_mutex_try(sqlite3_mutex *p){ */ void sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -149,9 +157,11 @@ void sqlite3_mutex_leave(sqlite3_mutex *p){ ** intended for use inside assert() statements. */ int sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } int sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif diff --git a/src/mutex_unix.c b/src/mutex_unix.c index 0a493fa6a7..cebb96c90e 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -80,6 +80,17 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization only. +*/ +void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ diff --git a/src/mutex_w32.c b/src/mutex_w32.c index fc943acaa0..90be07db2d 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -77,6 +77,19 @@ static int winMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization only. +*/ +void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#else + MemoryBarrier(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ diff --git a/src/pager.c b/src/pager.c index 060edb8d1d..2f7c330d60 100644 --- a/src/pager.c +++ b/src/pager.c @@ -647,7 +647,7 @@ struct Pager { u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ - u8 hasBeenUsed; /* True if any content previously read */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -5097,10 +5097,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && pPager->hasBeenUsed ){ + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ /* The shared-lock has just been acquired then check to ** see if the database has been modified. If the database has changed, - ** flush the cache. The pPager->hasBeenUsed flag prevents this from + ** flush the cache. The hasHeldSharedLock flag prevents this from ** occurring on the very first access to a file, in order to save a ** single unnecessary sqlite3OsRead() call at the start-up. ** @@ -5170,6 +5170,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; } return rc; } @@ -5253,21 +5254,25 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + const int bMmapOk = (pgno>1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } - pPager->hasBeenUsed = 1; + assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -5422,7 +5427,7 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); - assert( pPage==0 || pPager->hasBeenUsed ); + assert( pPage==0 || pPager->hasHeldSharedLock ); if( pPage==0 ) return 0; return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } @@ -6389,7 +6394,7 @@ u8 sqlite3PagerIsreadonly(Pager *pPager){ #ifdef SQLITE_DEBUG /* -** Return the number of references to the pager. +** Return the sum of the reference counts for all pages held by pPager. */ int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); diff --git a/src/pcache.c b/src/pcache.c index 58c05ac2a4..e39262cb8c 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -19,7 +19,7 @@ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ + int nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ @@ -184,7 +184,7 @@ int sqlite3PcacheOpen( ** are no outstanding page references when this function is called. */ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( @@ -351,9 +351,7 @@ PgHdr *sqlite3PcacheFetchFinish( if( !pPgHdr->pPage ){ return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } + pCache->nRefSum++; pPgHdr->nRef++; return pPgHdr; } @@ -364,9 +362,8 @@ PgHdr *sqlite3PcacheFetchFinish( */ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - p->pCache->nRef--; + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); }else if( p->pDirtyPrev!=0 ){ @@ -382,6 +379,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); p->nRef++; + p->pCache->nRefSum++; } /* @@ -394,7 +392,7 @@ void sqlite3PcacheDrop(PgHdr *p){ if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - p->pCache->nRef--; + p->pCache->nRefSum--; sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } @@ -490,11 +488,11 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->nRef ){ + if( pgno==0 && pCache->nRefSum ){ sqlite3_pcache_page *pPage1; pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because - ** pCache->nRef>0 */ + ** pCache->nRefSum>0 */ memset(pPage1->pBuf, 0, pCache->szPage); pgno = 1; } @@ -600,10 +598,13 @@ PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ } /* -** Return the total number of referenced pages held by the cache. +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. */ int sqlite3PcacheRefCount(PCache *pCache){ - return pCache->nRef; + return pCache->nRefSum; } /* diff --git a/src/pcache1.c b/src/pcache1.c index 18df8c87d3..00a0205f5a 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -87,6 +87,24 @@ typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ + u8 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -115,7 +133,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every @@ -153,23 +171,6 @@ struct PCache1 { void *pBulk; /* Bulk memory used by pcache-local */ }; -/* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - u8 isBulkLocal; /* This page from bulk local storage */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - /* ** Free slots in the allocator used to divide up the global page cache ** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. @@ -230,6 +231,7 @@ static SQLITE_WSD struct PCacheGlobal { /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -289,6 +291,7 @@ static int pcache1InitBulk(PCache1 *pCache){ pX->page.pBuf = zBulk; pX->page.pExtra = &pX[1]; pX->isBulkLocal = 1; + pX->isAnchor = 0; pX->pNext = pCache->pFree; pCache->pFree = pX; zBulk += pCache->szAlloc; @@ -431,6 +434,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; + p->isAnchor = 0; } if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage++; @@ -557,22 +561,16 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - assert( pPage->pLruNext || pPage==pCache->pGroup->pLruTail ); - assert( pPage->pLruPrev || pPage==pCache->pGroup->pLruHead ); + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - }else{ - pCache->pGroup->pLruHead = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - }else{ - pCache->pGroup->pLruTail = pPage->pLruPrev; - } + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; + assert( pPage->isAnchor==0 ); + assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; return pPage; } @@ -605,9 +603,11 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ */ static void pcache1EnforceMaxPage(PCache1 *pCache){ PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ - PgHdr1 *p = pGroup->pLruTail; + while( pGroup->nCurrentPage>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); @@ -741,6 +741,10 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ }else{ pGroup = &pcache1.grp; } + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; @@ -848,11 +852,11 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( /* Step 4. Try to recycle a page. */ if( pCache->bPurgeable - && pGroup->pLruTail + && !pGroup->lru.pLruPrev->isAnchor && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) ){ PCache1 *pOther; - pPage = pGroup->pLruTail; + pPage = pGroup->lru.pLruPrev; assert( pPage->isPinned==0 ); pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); @@ -961,7 +965,10 @@ static PgHdr1 *pcache1FetchNoMutex( pPage = pCache->apHash[iKey % pCache->nHash]; while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } - /* Step 2: Abort if no existing page is found and createFlag is 0 */ + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ if( pPage ){ if( !pPage->isPinned ){ return pcache1PinPage(pPage); @@ -1038,21 +1045,16 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ - if( pGroup->pLruHead ){ - pGroup->pLruHead->pLruPrev = pPage; - pPage->pLruNext = pGroup->pLruHead; - pGroup->pLruHead = pPage; - }else{ - pGroup->pLruTail = pPage; - pGroup->pLruHead = pPage; - } + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; pCache->nRecyclable++; pPage->isPinned = 0; } @@ -1190,7 +1192,10 @@ int sqlite3PcacheReleaseMemory(int nReq){ if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFreeisAnchor==0 + ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); @@ -1218,7 +1223,7 @@ void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } diff --git a/src/resolve.c b/src/resolve.c index 04fa8429a8..2c4212ba7d 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -45,30 +45,6 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. -** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. -** -** The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x -** -** Is equivalent to: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 -** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. On the other hand Standard SQL does -** not allow the GROUP BY clause to contain references to result-set columns. -** So this should never come up in well-formed queries. -** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -102,19 +78,11 @@ static void resolveAlias( db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - incrAggFunctionDepth(pDup, nSubquery); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - ExprSetProperty(pDup, EP_Skip); - if( pEList->a[iCol].u.x.iAlias==0 ){ - pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].u.x.iAlias; - } + if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -506,7 +474,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( pExpr->op!=TK_AS ){ + if( !ExprHasProperty(pExpr, EP_Alias) ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to @@ -547,36 +515,25 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ } /* -** Report an error that an expression is not valid for a partial index WHERE -** clause. +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. */ -static void notValidPartIdxWhere( +static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ + const char *zMsg, /* Type of error */ + int validMask /* Set of contexts for which prohibited */ ){ - if( (pNC->ncFlags & NC_PartIdx)!=0 ){ - sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", - zMsg); - } -} - + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + if( (pNC->ncFlags & validMask)!=0 ){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK -/* -** Report an error that an expression is not valid for a CHECK constraint. -*/ -static void notValidCheckConstraint( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ -){ - if( (pNC->ncFlags & NC_IsCheck)!=0 ){ - sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } -#else -# define notValidCheckConstraint(P,N,M) -#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -661,6 +618,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -690,7 +649,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValidPartIdxWhere(pParse, pNC, "functions"); + notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); @@ -738,9 +697,18 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query */ ExprSetProperty(pExpr,EP_ConstFunc); } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index. */ + notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); @@ -786,8 +754,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValidCheckConstraint(pParse, pNC, "subqueries"); - notValidPartIdxWhere(pParse, pNC, "subqueries"); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -797,8 +764,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValidCheckConstraint(pParse, pNC, "parameters"); - notValidPartIdxWhere(pParse, pNC, "parameters"); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } } @@ -1501,14 +1467,14 @@ void sqlite3ResolveSelectNames( void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx */ + int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - assert( type==NC_IsCheck || type==NC_PartIdx ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; diff --git a/src/shell.c b/src/shell.c index e04fe784a8..e5eb394ead 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4252,8 +4252,8 @@ static int process_input(ShellState *p, FILE *in){ fprintf(stderr, "Error: incomplete SQL: %s\n", zSql); errCnt++; } - free(zSql); } + free(zSql); free(zLine); return errCnt>0; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a7d6318e5b..7a51ec109f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -124,7 +124,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** ** 
@@ -374,7 +374,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** Restrictions:
 **
 ** 
    
-** 
  •  The application must insure that the 1st parameter to sqlite3_exec()
+** 
  •  The application must ensure that the 1st parameter to sqlite3_exec()
 **      is a valid and open [database connection].
 ** 
  •  The application must not close the [database connection] specified by
 **      the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
@@ -1367,9 +1367,11 @@ int sqlite3_os_end(void);
 ** applications and so this routine is usually not necessary.  It is
 ** provided to support rare applications with unusual needs.
 **
-** The sqlite3_config() interface is not threadsafe.  The application
-** must insure that no other SQLite interfaces are invoked by other
-** threads while sqlite3_config() is running.  Furthermore, sqlite3_config()
+** The sqlite3_config() interface is not threadsafe. The application
+** must ensure that no other SQLite interfaces are invoked by other
+** threads while sqlite3_config() is running.
+**
+** The sqlite3_config() interface
 ** may only be invoked prior to library initialization using
 ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
 ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
@@ -4357,6 +4359,22 @@ const void *sqlite3_value_text16be(sqlite3_value*);
 int sqlite3_value_type(sqlite3_value*);
 int sqlite3_value_numeric_type(sqlite3_value*);
 
+/*
+** CAPI3REF: Obtaining SQL Values
+** METHOD: sqlite3_value
+**
+** The sqlite3_value_subtype(V) function returns the subtype for
+** an [application-defined SQL function] argument V.  The subtype
+** information can be used to pass a limited amount of context from
+** one SQL function to another.  Use the [sqlite3_result_subtype()]
+** routine to set the subtype for the return value of an SQL function.
+**
+** SQLite makes no use of subtype itself.  It merely passes the subtype
+** from the result of one [application-defined SQL function] into the
+** input of another.
+*/
+unsigned int sqlite3_value_subtype(sqlite3_value*);
+
 /*
 ** CAPI3REF: Copy And Free SQL Values
 ** METHOD: sqlite3_value
@@ -4656,6 +4674,21 @@ void sqlite3_result_value(sqlite3_context*, sqlite3_value*);
 void sqlite3_result_zeroblob(sqlite3_context*, int n);
 int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n);
 
+
+/*
+** CAPI3REF: Setting The Subtype Of An SQL Function
+** METHOD: sqlite3_context
+**
+** The sqlite3_result_subtype(C,T) function causes the subtype of
+** the result from the [application-defined SQL function] with 
+** [sqlite3_context] C to be the value T.  Only the lower 8 bits 
+** of the subtype T are preserved in current versions of SQLite;
+** higher order bits are discarded.
+** The number of subtype bytes preserved by SQLite might increase
+** in future releases of SQLite.
+*/
+void sqlite3_result_subtype(sqlite3_context*,unsigned int);
+
 /*
 ** CAPI3REF: Define New Collating Sequences
 ** METHOD: sqlite3
@@ -6094,6 +6127,9 @@ int sqlite3_vfs_unregister(sqlite3_vfs*);
 ** 
  •   SQLITE_MUTEX_STATIC_APP1
 ** 
  •   SQLITE_MUTEX_STATIC_APP2
 ** 
  •   SQLITE_MUTEX_STATIC_APP3
+** 
  •   SQLITE_MUTEX_STATIC_VFS1
+** 
  •   SQLITE_MUTEX_STATIC_VFS2
+** 
  •   SQLITE_MUTEX_STATIC_VFS3
 ** 

 **
 ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE)
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 1525327f49..9b9f610e93 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -272,6 +272,9 @@ struct sqlite3_api_routines {
   void (*value_free)(sqlite3_value*);
   int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64);
   int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64);
+  /* Version 3.8.12 and later */
+  unsigned int (*value_subtype)(sqlite3_value*);
+  void (*result_subtype)(sqlite3_context*,unsigned int);
 };
 
 /*
@@ -285,7 +288,7 @@ struct sqlite3_api_routines {
 ** the API.  So the redefinition macros are only valid if the
 ** SQLITE_CORE macros is undefined.
 */
-#ifndef SQLITE_CORE
+#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
 #define sqlite3_aggregate_context      sqlite3_api->aggregate_context
 #ifndef SQLITE_OMIT_DEPRECATED
 #define sqlite3_aggregate_count        sqlite3_api->aggregate_count
@@ -508,9 +511,12 @@ struct sqlite3_api_routines {
 #define sqlite3_value_free             sqlite3_api->value_free
 #define sqlite3_result_zeroblob64      sqlite3_api->result_zeroblob64
 #define sqlite3_bind_zeroblob64        sqlite3_api->bind_zeroblob64
-#endif /* SQLITE_CORE */
+/* Version 3.8.12 and later */
+#define sqlite3_value_subtype          sqlite3_api->value_subtype
+#define sqlite3_result_subtype         sqlite3_api->result_subtype
+#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
-#ifndef SQLITE_CORE
+#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
   /* This case when the file really is being compiled as a loadable 
   ** extension */
 # define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api=0;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 2adb781013..4a9b135312 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1385,18 +1385,20 @@ struct FuncDestructor {
 ** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG.  There
 ** are assert() statements in the code to verify this.
 */
-#define SQLITE_FUNC_ENCMASK  0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */
-#define SQLITE_FUNC_LIKE     0x004 /* Candidate for the LIKE optimization */
-#define SQLITE_FUNC_CASE     0x008 /* Case-sensitive LIKE-type function */
-#define SQLITE_FUNC_EPHEM    0x010 /* Ephemeral.  Delete with VDBE */
-#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */
-#define SQLITE_FUNC_LENGTH   0x040 /* Built-in length() function */
-#define SQLITE_FUNC_TYPEOF   0x080 /* Built-in typeof() function */
-#define SQLITE_FUNC_COUNT    0x100 /* Built-in count(*) aggregate */
-#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */
-#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */
-#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */
-#define SQLITE_FUNC_MINMAX  0x1000 /* True for min() and max() aggregates */
+#define SQLITE_FUNC_ENCMASK  0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */
+#define SQLITE_FUNC_LIKE     0x0004 /* Candidate for the LIKE optimization */
+#define SQLITE_FUNC_CASE     0x0008 /* Case-sensitive LIKE-type function */
+#define SQLITE_FUNC_EPHEM    0x0010 /* Ephemeral.  Delete with VDBE */
+#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/
+#define SQLITE_FUNC_LENGTH   0x0040 /* Built-in length() function */
+#define SQLITE_FUNC_TYPEOF   0x0080 /* Built-in typeof() function */
+#define SQLITE_FUNC_COUNT    0x0100 /* Built-in count(*) aggregate */
+#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */
+#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */
+#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */
+#define SQLITE_FUNC_MINMAX   0x1000 /* True for min() and max() aggregates */
+#define SQLITE_FUNC_SLOCHNG  0x2000 /* "Slow Change". Value constant during a
+                                    ** single query - might change over time */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
@@ -1412,6 +1414,12 @@ struct FuncDestructor {
 **   VFUNCTION(zName, nArg, iArg, bNC, xFunc)
 **     Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag.
 **
+**   DFUNCTION(zName, nArg, iArg, bNC, xFunc)
+**     Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and
+**     adds the SQLITE_FUNC_SLOCHNG flag.  Used for date & time functions
+**     and functions like sqlite_version() that can change, but not during
+**     a single query.
+**
 **   AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal)
 **     Used to create an aggregate function definition implemented by
 **     the C functions xStep and xFinal. The first four parameters
@@ -1432,11 +1440,14 @@ struct FuncDestructor {
 #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
    SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
+#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \
+  {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
 #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \
   {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\
    SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
 #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
-  {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
+  {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
    pArg, 0, xFunc, 0, 0, #zName, 0, 0}
 #define LIKEFUNC(zName, nArg, arg, flags) \
   {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \
@@ -1872,6 +1883,7 @@ struct Index {
   u8 *aSortOrder;          /* for each column: True==DESC, False==ASC */
   char **azColl;           /* Array of collation sequence names for index */
   Expr *pPartIdxWhere;     /* WHERE clause for partial indices */
+  ExprList *aColExpr;      /* Column expressions */
   int tnum;                /* DB Page containing root of this index */
   LogEst szIdxRow;         /* Estimated average row size in bytes */
   u16 nKeyCol;             /* Number of columns forming the key */
@@ -2121,9 +2133,10 @@ struct Expr {
 #define EP_MemToken  0x010000 /* Need to sqlite3DbFree() Expr.zToken */
 #define EP_NoReduce  0x020000 /* Cannot EXPRDUP_REDUCE this Expr */
 #define EP_Unlikely  0x040000 /* unlikely() or likelihood() function */
-#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */
+#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */
 #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */
 #define EP_Subquery  0x200000 /* Tree contains a TK_SELECT operator */
+#define EP_Alias     0x400000 /* Is an alias for a result set column */
 
 /*
 ** Combinations of two or more EP_* flags
@@ -2340,6 +2353,7 @@ struct SrcList {
 #define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
 #define WHERE_SORTBYGROUP      0x0800 /* Support sqlite3WhereIsSorted() */
 #define WHERE_REOPEN_IDX       0x1000 /* Try to use OP_ReopenIdx */
+#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */
 
 /* Allowed return values from sqlite3WhereIsDistinct()
 */
@@ -2392,6 +2406,7 @@ struct NameContext {
 #define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
 #define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
 #define NC_PartIdx   0x0010  /* True if resolving a partial index WHERE */
+#define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
 #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
 
 /*
@@ -2661,7 +2676,7 @@ struct Parse {
   int nOpAlloc;        /* Number of slots allocated for Vdbe.aOp[] */
   int iFixedOp;        /* Never back out opcodes iFixedOp-1 or earlier */
   int ckBase;          /* Base register of data during check constraints */
-  int iPartIdxTab;     /* Table corresponding to a partial index */
+  int iSelfTab;        /* Table of an index whose exprs are being coded */
   int iCacheLevel;     /* ColCache valid when aColCache[].iLevel<=iCacheLevel */
   int iCacheCnt;       /* Counter used to generate aColCache[].lru values */
   int nLabel;          /* Number of labels used */
@@ -3181,6 +3196,11 @@ const sqlite3_mem_methods *sqlite3MemGetMemsys5(void);
   int sqlite3MutexInit(void);
   int sqlite3MutexEnd(void);
 #endif
+#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP)
+  void sqlite3MemoryBarrier(void);
+#else
+# define sqlite3MemoryBarrier()
+#endif
 
 sqlite3_int64 sqlite3StatusValue(int);
 void sqlite3StatusUp(int, int);
@@ -3364,6 +3384,10 @@ int sqlite3WhereIsSorted(WhereInfo*);
 int sqlite3WhereContinueLabel(WhereInfo*);
 int sqlite3WhereBreakLabel(WhereInfo*);
 int sqlite3WhereOkOnePass(WhereInfo*, int*);
+#define ONEPASS_OFF      0        /* Use of ONEPASS not allowed */
+#define ONEPASS_SINGLE   1        /* ONEPASS valid for a single row update */
+#define ONEPASS_MULTI    2        /* ONEPASS is valid for multiple rows */
+void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int);
 int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8);
 void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
 void sqlite3ExprCodeMove(Parse*, int, int, int);
@@ -3426,8 +3450,9 @@ int sqlite3ExprIsInteger(Expr*, int*);
 int sqlite3ExprCanBeNull(const Expr*);
 int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
 int sqlite3IsRowid(const char*);
-void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8);
-void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*);
+void sqlite3GenerateRowDelete(
+    Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
+void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
 int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
 void sqlite3ResolvePartIdxLabel(Parse*,int);
 void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
diff --git a/src/test_func.c b/src/test_func.c
index 2e34fa074e..63cf18e3f7 100644
--- a/src/test_func.c
+++ b/src/test_func.c
@@ -462,7 +462,7 @@ static void real2hex(
 }
 
 /*
-** tclcmd: test_extract(record, field)
+**     test_extract(record, field)
 **
 ** This function implements an SQL user-function that accepts a blob
 ** containing a formatted database record as the first argument. The
@@ -509,7 +509,7 @@ static void test_extract(
 }
 
 /*
-** tclcmd: test_decode(record)
+**      test_decode(record)
 **
 ** This function implements an SQL user-function that accepts a blob
 ** containing a formatted database record as its only argument. It returns
@@ -601,6 +601,8 @@ static void test_decode(
 }
 
 /*
+**       test_zeroblob(N)
+**
 ** The implementation of scalar SQL function "test_zeroblob()". This is
 ** similar to the built-in zeroblob() function, except that it does not
 ** check that the integer parameter is within range before passing it
@@ -615,6 +617,31 @@ static void test_zeroblob(
   sqlite3_result_zeroblob(context, nZero);
 }
 
+/*         test_getsubtype(V)
+**
+** Return the subtype for value V.
+*/
+static void test_getsubtype(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_result_int(context, (int)sqlite3_value_subtype(argv[0]));
+}
+
+/*         test_setsubtype(V, T)
+**
+** Return the value V with its subtype changed to T
+*/
+static void test_setsubtype(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_result_value(context, argv[0]);
+  sqlite3_result_subtype(context, (unsigned int)sqlite3_value_int(argv[1]));
+}
+
 static int registerTestFunctions(sqlite3 *db){
   static const struct {
      char *zName;
@@ -641,6 +668,8 @@ static int registerTestFunctions(sqlite3 *db){
     { "test_decode",           1, SQLITE_UTF8, test_decode},
     { "test_extract",          2, SQLITE_UTF8, test_extract},
     { "test_zeroblob",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC, test_zeroblob},
+    { "test_getsubtype",       1, SQLITE_UTF8, test_getsubtype},
+    { "test_setsubtype",       2, SQLITE_UTF8, test_setsubtype},
   };
   int i;
 
diff --git a/src/tokenize.c b/src/tokenize.c
index 3d08f75a2a..6b5ad27901 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -403,6 +403,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   pParse->zTail = zSql;
   i = 0;
   assert( pzErrMsg!=0 );
+  /* sqlite3ParserTrace(stdout, "parser: "); */
   pEngine = sqlite3ParserAlloc(sqlite3Malloc);
   if( pEngine==0 ){
     db->mallocFailed = 1;
diff --git a/src/treeview.c b/src/treeview.c
index 59c07e4742..971de4e8bc 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -253,11 +253,6 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable);
       break;
     }
-    case TK_AS: {
-      sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken);
-      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
-      break;
-    }
     case TK_ID: {
       sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken);
       break;
diff --git a/src/update.c b/src/update.c
index f7b4dcd939..94f7a4dd99 100644
--- a/src/update.c
+++ b/src/update.c
@@ -272,7 +272,9 @@ void sqlite3Update(
 
   /* There is one entry in the aRegIdx[] array for each index on the table
   ** being updated.  Fill in aRegIdx[] with a register number that will hold
-  ** the key for accessing each index.  
+  ** the key for accessing each index.
+  **
+  ** FIXME:  Be smarter about omitting indexes that use expressions.
   */
   for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
     int reg;
@@ -281,7 +283,8 @@ void sqlite3Update(
     }else{
       reg = 0;
       for(i=0; inKeyCol; i++){
-        if( aXRef[pIdx->aiColumn[i]]>=0 ){
+        i16 iIdxCol = pIdx->aiColumn[i];
+        if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){
           reg = ++pParse->nMem;
           break;
         }
@@ -381,6 +384,7 @@ void sqlite3Update(
     if( pWInfo==0 ) goto update_cleanup;
     okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
     for(i=0; iaiColumn[i]>=(-1) );
       sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i],
                                       iPk+i);
     }
@@ -583,7 +587,7 @@ void sqlite3Update(
       }
       VdbeCoverageNeverTaken(v);
     }
-    sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx);
+    sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1);
   
     /* If changing the record number, delete the old record.  */
     if( hasFK || chngKey || pPk!=0 ){
diff --git a/src/vdbe.c b/src/vdbe.c
index caf3acabb9..41ad915b16 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3972,9 +3972,10 @@ case OP_Found: {        /* jump, in3 */
 **
 ** P1 is the index of a cursor open on an SQL table btree (with integer
 ** keys).  P3 is an integer rowid.  If P1 does not contain a record with
-** rowid P3 then jump immediately to P2.  If P1 does contain a record
-** with rowid P3 then leave the cursor pointing at that record and fall
-** through to the next instruction.
+** rowid P3 then jump immediately to P2.  Or, if P2 is 0, raise an
+** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then 
+** leave the cursor pointing at that record and fall through to the next
+** instruction.
 **
 ** The OP_NotFound opcode performs the same operation on index btrees
 ** (with arbitrary multi-value keys).
@@ -4006,13 +4007,21 @@ case OP_NotExists: {        /* jump, in3 */
   res = 0;
   iKey = pIn3->u.i;
   rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res);
+  assert( rc==SQLITE_OK || res==0 );
   pC->movetoTarget = iKey;  /* Used by OP_Delete */
   pC->nullRow = 0;
   pC->cacheStatus = CACHE_STALE;
   pC->deferredMoveto = 0;
   VdbeBranchTaken(res!=0,2);
   pC->seekResult = res;
-  if( res!=0 ) goto jump_to_p2;
+  if( res!=0 ){
+    assert( rc==SQLITE_OK );
+    if( pOp->p2==0 ){
+      rc = SQLITE_CORRUPT_BKPT;
+    }else{
+      goto jump_to_p2;
+    }
+  }
   break;
 }
 
@@ -4278,14 +4287,15 @@ case OP_InsertInt: {
   break;
 }
 
-/* Opcode: Delete P1 P2 * P4 *
+/* Opcode: Delete P1 P2 * P4 P5
 **
 ** Delete the record at which the P1 cursor is currently pointing.
 **
-** The cursor will be left pointing at either the next or the previous
-** record in the table. If it is left pointing at the next record, then
-** the next Next instruction will be a no-op.  Hence it is OK to delete
-** a record from within a Next loop.
+** If the P5 parameter is non-zero, the cursor will be left pointing at 
+** either the next or the previous record in the table. If it is left 
+** pointing at the next record, then the next Next instruction will be a 
+** no-op. As a result, in this case it is OK to delete a record from within a
+** Next loop. If P5 is zero, then the cursor is left in an undefined state.
 **
 ** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is
 ** incremented (otherwise not).
@@ -4300,6 +4310,7 @@ case OP_InsertInt: {
 */
 case OP_Delete: {
   VdbeCursor *pC;
+  u8 hasUpdateCallback;
 
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
@@ -4307,22 +4318,27 @@ case OP_Delete: {
   assert( pC->pCursor!=0 );  /* Only valid for real tables, no pseudotables */
   assert( pC->deferredMoveto==0 );
 
+  hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable;
+  if( pOp->p5 && hasUpdateCallback ){
+    sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget);
+  }
+
 #ifdef SQLITE_DEBUG
   /* The seek operation that positioned the cursor prior to OP_Delete will
   ** have also set the pC->movetoTarget field to the rowid of the row that
   ** is being deleted */
-  if( pOp->p4.z && pC->isTable ){
+  if( pOp->p4.z && pC->isTable && pOp->p5==0 ){
     i64 iKey = 0;
     sqlite3BtreeKeySize(pC->pCursor, &iKey);
     assert( pC->movetoTarget==iKey ); 
   }
 #endif
  
-  rc = sqlite3BtreeDelete(pC->pCursor);
+  rc = sqlite3BtreeDelete(pC->pCursor, pOp->p5);
   pC->cacheStatus = CACHE_STALE;
 
   /* Invoke the update-hook if required. */
-  if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
+  if( rc==SQLITE_OK && hasUpdateCallback ){
     db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE,
                         db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget);
     assert( pC->iDb>=0 );
@@ -4861,7 +4877,7 @@ case OP_IdxDelete: {
 #endif
   rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res);
   if( rc==SQLITE_OK && res==0 ){
-    rc = sqlite3BtreeDelete(pCrsr);
+    rc = sqlite3BtreeDelete(pCrsr, 0);
   }
   assert( pC->deferredMoveto==0 );
   pC->cacheStatus = CACHE_STALE;
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index b5470c2dda..45892a899d 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -186,6 +186,7 @@ struct Mem {
   } u;
   u16 flags;          /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */
   u8  enc;            /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */
+  u8  eSubtype;       /* Subtype for this value */
   int n;              /* Number of characters in string value, excluding '\0' */
   char *z;            /* String or BLOB value */
   /* ShallowCopy only needs to copy the information above */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 15a8bba0d4..06b14e1276 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -187,6 +187,9 @@ int sqlite3_value_int(sqlite3_value *pVal){
 sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){
   return sqlite3VdbeIntValue((Mem*)pVal);
 }
+unsigned int sqlite3_value_subtype(sqlite3_value *pVal){
+  return ((Mem*)pVal)->eSubtype;
+}
 const unsigned char *sqlite3_value_text(sqlite3_value *pVal){
   return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8);
 }
@@ -365,6 +368,10 @@ void sqlite3_result_null(sqlite3_context *pCtx){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetNull(pCtx->pOut);
 }
+void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){
+  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
+  pCtx->pOut->eSubtype = eSubtype & 0xff;
+}
 void sqlite3_result_text(
   sqlite3_context *pCtx, 
   const char *z, 
@@ -696,7 +703,7 @@ void *sqlite3_user_data(sqlite3_context *p){
 ** application defined function.
 */
 sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){
-  assert( p && p->pFunc );
+  assert( p && p->pOut );
   return p->pOut->db;
 }
 
diff --git a/src/vdbeblob.c b/src/vdbeblob.c
index ea01f5ce80..2cdc3edb00 100644
--- a/src/vdbeblob.c
+++ b/src/vdbeblob.c
@@ -247,7 +247,8 @@ int sqlite3_blob_open(
       for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
         int j;
         for(j=0; jnKeyCol; j++){
-          if( pIdx->aiColumn[j]==iCol ){
+          /* FIXME: Be smarter about indexes that use expressions */
+          if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==(-2) ){
             zFault = "indexed";
           }
         }
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 648a53d2a3..28dd5d9572 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1155,7 +1155,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){
 ** to be a scalar SQL function. If
 **
 **   * all function arguments are SQL literals,
-**   * the SQLITE_FUNC_CONSTANT function flag is set, and
+**   * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and
 **   * the SQLITE_FUNC_NEEDCOLL function flag is not set,
 **
 ** then this routine attempts to invoke the SQL function. Assuming no
@@ -1196,7 +1196,7 @@ static int valueFromFunction(
   nName = sqlite3Strlen30(p->u.zToken);
   pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0);
   assert( pFunc );
-  if( (pFunc->funcFlags & SQLITE_FUNC_CONSTANT)==0 
+  if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 
    || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
   ){
     return SQLITE_OK;
diff --git a/src/wal.c b/src/wal.c
index cf8f1d4e66..d87d2c17ce 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2462,7 +2462,8 @@ int sqlite3WalFindFrame(
   {
     u32 iRead2 = 0;
     u32 iTest;
-    for(iTest=iLast; iTest>0; iTest--){
+    assert( pWal->minFrame>0 );
+    for(iTest=iLast; iTest>=pWal->minFrame; iTest--){
       if( walFramePgno(pWal, iTest)==pgno ){
         iRead2 = iTest;
         break;
diff --git a/src/where.c b/src/where.c
index de8a041e7c..fc849ccdb8 100644
--- a/src/where.c
+++ b/src/where.c
@@ -69,9 +69,11 @@ int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
 }
 
 /*
-** Return TRUE if an UPDATE or DELETE statement can operate directly on
-** the rowids returned by a WHERE clause.  Return FALSE if doing an
-** UPDATE or DELETE might change subsequent WHERE clause results.
+** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
+** operate directly on the rowis returned by a WHERE clause.  Return
+** ONEPASS_SINGLE (1) if the statement can operation directly because only
+** a single row is to be changed.  Return ONEPASS_MULTI (2) if the one-pass
+** optimization can be used on multiple 
 **
 ** If the ONEPASS optimization is used (if this routine returns true)
 ** then also write the indices of open cursors used by ONEPASS
@@ -85,7 +87,7 @@ int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
 */
 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
   memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
-  return pWInfo->okOnePass;
+  return pWInfo->eOnePass;
 }
 
 /*
@@ -180,10 +182,13 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
   while( pScan->iEquiv<=pScan->nEquiv ){
     iCur = pScan->aiCur[pScan->iEquiv-1];
     iColumn = pScan->aiColumn[pScan->iEquiv-1];
+    assert( iColumn!=(-2) || pScan->pIdxExpr!=0 );
     while( (pWC = pScan->pWC)!=0 ){
       for(pTerm=pWC->a+k; knTerm; k++, pTerm++){
         if( pTerm->leftCursor==iCur
          && pTerm->u.leftColumn==iColumn
+         && (iColumn!=(-2)
+               || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0)
          && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin))
         ){
           if( (pTerm->eOperator & WO_EQUIV)!=0
@@ -268,16 +273,19 @@ static WhereTerm *whereScanInit(
   u32 opMask,             /* Operator(s) to scan for */
   Index *pIdx             /* Must be compatible with this index */
 ){
-  int j;
+  int j = 0;
 
   /* memset(pScan, 0, sizeof(*pScan)); */
   pScan->pOrigWC = pWC;
   pScan->pWC = pWC;
+  pScan->pIdxExpr = 0;
+  if( pIdx ){
+    j = iColumn;
+    iColumn = pIdx->aiColumn[j];
+    if( iColumn==(-2) ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
+  }
   if( pIdx && iColumn>=0 ){
     pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
-    for(j=0; pIdx->aiColumn[j]!=iColumn; j++){
-      if( NEVER(j>pIdx->nColumn) ) return 0;
-    }
     pScan->zCollName = pIdx->azColl[j];
   }else{
     pScan->idxaff = 0;
@@ -298,6 +306,9 @@ static WhereTerm *whereScanInit(
 ** the WO_xx operator codes specified by the op parameter.
 ** Return a pointer to the term.  Return 0 if not found.
 **
+** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx
+** rather than the iColumn-th column of table iCur.
+**
 ** The term returned might by Y= if there is another constraint in
 ** the WHERE clause that specifies that X=Y.  Any such constraints will be
 ** identified by the WO_EQUIV bit in the pTerm->eOperator field.  The
@@ -373,6 +384,25 @@ static int findIndexCol(
   return -1;
 }
 
+/*
+** Return TRUE if the iCol-th column of index pIdx is NOT NULL
+*/
+static int indexColumnNotNull(Index *pIdx, int iCol){
+  int j;
+  assert( pIdx!=0 );
+  assert( iCol>=0 && iColnColumn );
+  j = pIdx->aiColumn[iCol];
+  if( j>=0 ){
+    return pIdx->pTable->aCol[j].notNull;
+  }else if( j==(-1) ){
+    return 1;
+  }else{
+    assert( j==(-2) );
+    return 0;  /* Assume an indexed expression can always yield a NULL */
+
+  }
+}
+
 /*
 ** Return true if the DISTINCT expression-list passed as the third argument
 ** is redundant.
@@ -423,12 +453,9 @@ static int isDistinctRedundant(
   for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
     if( !IsUniqueIndex(pIdx) ) continue;
     for(i=0; inKeyCol; i++){
-      i16 iCol = pIdx->aiColumn[i];
-      if( 0==sqlite3WhereFindTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){
-        int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i);
-        if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){
-          break;
-        }
+      if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
+        if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
+        if( indexColumnNotNull(pIdx, i)==0 ) break;
       }
     }
     if( i==pIdx->nKeyCol ){
@@ -780,6 +807,7 @@ static sqlite3_index_info *allocateIndexInfo(
     testcase( pTerm->eOperator & WO_ALL );
     if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue;
     if( pTerm->wtFlags & TERM_VNULL ) continue;
+    assert( pTerm->u.leftColumn>=(-1) );
     nTerm++;
   }
 
@@ -835,6 +863,7 @@ static sqlite3_index_info *allocateIndexInfo(
     testcase( pTerm->eOperator & WO_ALL );
     if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue;
     if( pTerm->wtFlags & TERM_VNULL ) continue;
+    assert( pTerm->u.leftColumn>=(-1) );
     pIdxCons[j].iColumn = pTerm->u.leftColumn;
     pIdxCons[j].iTermOffset = i;
     op = (u8)pTerm->eOperator & WO_ALL;
@@ -2126,7 +2155,6 @@ static int whereLoopAddBtreeIndex(
   u16 saved_nSkip;                /* Original value of pNew->nSkip */
   u32 saved_wsFlags;              /* Original value of pNew->wsFlags */
   LogEst saved_nOut;              /* Original value of pNew->nOut */
-  int iCol;                       /* Index of the column in the table */
   int rc = SQLITE_OK;             /* Return code */
   LogEst rSize;                   /* Number of rows in the table */
   LogEst rLogSize;                /* Logarithm of table size */
@@ -2147,16 +2175,15 @@ static int whereLoopAddBtreeIndex(
   if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
 
   assert( pNew->u.btree.nEqnColumn );
-  iCol = pProbe->aiColumn[pNew->u.btree.nEq];
 
-  pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol,
-                        opMask, pProbe);
   saved_nEq = pNew->u.btree.nEq;
   saved_nSkip = pNew->nSkip;
   saved_nLTerm = pNew->nLTerm;
   saved_wsFlags = pNew->wsFlags;
   saved_prereq = pNew->prereq;
   saved_nOut = pNew->nOut;
+  pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
+                        opMask, pProbe);
   pNew->rSetup = 0;
   rSize = pProbe->aiRowLogEst[0];
   rLogSize = estLog(rSize);
@@ -2169,7 +2196,7 @@ static int whereLoopAddBtreeIndex(
     int nRecValid = pBuilder->nRecValid;
 #endif
     if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
-     && (iCol<0 || pSrc->pTab->aCol[iCol].notNull)
+     && indexColumnNotNull(pProbe, saved_nEq)
     ){
       continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
     }
@@ -2206,8 +2233,10 @@ static int whereLoopAddBtreeIndex(
                         ** changes "x IN (?)" into "x=?". */
 
     }else if( eOp & (WO_EQ|WO_IS) ){
+      int iCol = pProbe->aiColumn[saved_nEq];
       pNew->wsFlags |= WHERE_COLUMN_EQ;
-      if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){
+      assert( saved_nEq==pNew->u.btree.nEq );
+      if( iCol==(-1) || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){
         if( iCol>=0 && pProbe->uniqNotNull==0 ){
           pNew->wsFlags |= WHERE_UNQ_WANTED;
         }else{
@@ -2258,7 +2287,7 @@ static int whereLoopAddBtreeIndex(
       assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
 
       assert( pNew->nOut==saved_nOut );
-      if( pTerm->truthProb<=0 && iCol>=0 ){
+      if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
         assert( (eOp & WO_IN) || nIn==0 );
         testcase( eOp & WO_IN );
         pNew->nOut += pTerm->truthProb;
@@ -3785,7 +3814,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
       ) continue;
       opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
       for(j=0; jnKeyCol; j++){
-        pTerm = sqlite3WhereFindTerm(pWC, iCur, pIdx->aiColumn[j], 0, opMask, pIdx);
+        pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx);
         if( pTerm==0 ) break;
         testcase( pTerm->eOperator & WO_IS );
         pLoop->aLTerm[j] = pTerm;
@@ -3931,6 +3960,10 @@ WhereInfo *sqlite3WhereBegin(
   sqlite3 *db;               /* Database connection */
   int rc;                    /* Return code */
 
+  assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
+        (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
+     && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 
+  ));
 
   /* Variable initialization */
   db = pParse->db;
@@ -3986,6 +4019,7 @@ WhereInfo *sqlite3WhereBegin(
   pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v);
   pWInfo->wctrlFlags = wctrlFlags;
   pWInfo->savedNQueryLoop = pParse->nQueryLoop;
+  assert( pWInfo->eOnePass==ONEPASS_OFF );  /* ONEPASS defaults to OFF */
   pMaskSet = &pWInfo->sMaskSet;
   sWLB.pWInfo = pWInfo;
   sWLB.pWC = &pWInfo->sWC;
@@ -4025,14 +4059,12 @@ WhereInfo *sqlite3WhereBegin(
 
   /* Assign a bit from the bitmask to every term in the FROM clause.
   **
-  ** When assigning bitmask values to FROM clause cursors, it must be
-  ** the case that if X is the bitmask for the N-th FROM clause term then
-  ** the bitmask for all FROM clause terms to the left of the N-th term
-  ** is (X-1).   An expression from the ON clause of a LEFT JOIN can use
-  ** its Expr.iRightJoinTable value to find the bitmask of the right table
-  ** of the join.  Subtracting one from the right table bitmask gives a
-  ** bitmask for all tables to the left of the join.  Knowing the bitmask
-  ** for all tables to the left of a left join is important.  Ticket #3015.
+  ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in
   ** pTabList, not just the first nTabList tables.  nTabList is normally
@@ -4043,14 +4075,10 @@ WhereInfo *sqlite3WhereBegin(
     createMask(pMaskSet, pTabList->a[ii].iCursor);
     sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
   }
-#ifndef NDEBUG
-  {
-    Bitmask toTheLeft = 0;
-    for(ii=0; iinSrc; ii++){
-      Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
-      assert( (m-1)==toTheLeft );
-      toTheLeft |= m;
-    }
+#ifdef SQLITE_DEBUG
+  for(ii=0; iinSrc; ii++){
+    Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
+    assert( m==MASKBIT(ii) );
   }
 #endif
 
@@ -4178,11 +4206,16 @@ WhereInfo *sqlite3WhereBegin(
   ** the statement to update or delete a single row.
   */
   assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
-  if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
-   && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){
-    pWInfo->okOnePass = 1;
-    if( HasRowid(pTabList->a[0].pTab) ){
-      pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY;
+  if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
+    int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
+    int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
+    if( bOnerow || ( (wctrlFlags & WHERE_ONEPASS_MULTIROW)
+       && 0==(wsFlags & WHERE_VIRTUALTABLE)
+    )){
+      pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
+      if( HasRowid(pTabList->a[0].pTab) ){
+        pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY;
+      }
     }
   }
 
@@ -4213,15 +4246,15 @@ WhereInfo *sqlite3WhereBegin(
     if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
          && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){
       int op = OP_OpenRead;
-      if( pWInfo->okOnePass ){
+      if( pWInfo->eOnePass!=ONEPASS_OFF ){
         op = OP_OpenWrite;
         pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
       };
       sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
       assert( pTabItem->iCursor==pLevel->iTabCur );
-      testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 );
-      testcase( !pWInfo->okOnePass && pTab->nCol==BMS );
-      if( !pWInfo->okOnePass && pTab->nColeOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
+      testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
+      if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColcolUsed;
         int n = 0;
         for(; b; b=b>>1, n++){}
@@ -4252,7 +4285,7 @@ WhereInfo *sqlite3WhereBegin(
         ** WITHOUT ROWID table.  No need for a separate index */
         iIndexCur = pLevel->iTabCur;
         op = 0;
-      }else if( pWInfo->okOnePass ){
+      }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
         Index *pJ = pTabItem->pTab->pIndex;
         iIndexCur = iIdxCur;
         assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
@@ -4460,7 +4493,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
      && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0
     ){
       int ws = pLoop->wsFlags;
-      if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){
+      if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){
         sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor);
       }
       if( (ws & WHERE_INDEXED)!=0
diff --git a/src/whereInt.h b/src/whereInt.h
index 7138b85b25..cae09acc82 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -286,6 +286,7 @@ struct WhereScan {
   WhereClause *pOrigWC;      /* Original, innermost WhereClause */
   WhereClause *pWC;          /* WhereClause currently being scanned */
   char *zCollName;           /* Required collating sequence, if not NULL */
+  Expr *pIdxExpr;            /* Search for this index expression */
   char idxaff;               /* Must match this affinity, if zCollName!=NULL */
   unsigned char nEquiv;      /* Number of entries in aEquiv[] */
   unsigned char iEquiv;      /* Next unused slot in aEquiv[] */
@@ -411,7 +412,7 @@ struct WhereInfo {
   u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
   i8 nOBSat;                /* Number of ORDER BY terms satisfied by indices */
   u8 sorted;                /* True if really sorted (not just grouped) */
-  u8 okOnePass;             /* Ok to use one-pass algorithm for UPDATE/DELETE */
+  u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
   u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
   u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
   u8 nLevel;                /* Number of nested loop */
diff --git a/src/wherecode.c b/src/wherecode.c
index e5e0c0a6e6..ec24ba0ac0 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -41,6 +41,16 @@ static void explainAppendTerm(
   sqlite3StrAccumAppend(pStr, "?", 1);
 }
 
+/*
+** Return the name of the i-th column of the pIdx index.
+*/
+static const char *explainIndexColumnName(Index *pIdx, int i){
+  i = pIdx->aiColumn[i];
+  if( i==(-2) ) return "";
+  if( i==(-1) ) return "rowid";
+  return pIdx->pTable->aCol[i].zName;
+}
+
 /*
 ** Argument pLevel describes a strategy for scanning table pTab. This 
 ** function appends text to pStr that describes the subset of table
@@ -60,24 +70,22 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){
   u16 nEq = pLoop->u.btree.nEq;
   u16 nSkip = pLoop->nSkip;
   int i, j;
-  Column *aCol = pTab->aCol;
-  i16 *aiColumn = pIndex->aiColumn;
 
   if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return;
   sqlite3StrAccumAppend(pStr, " (", 2);
   for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z);
   }
 
   j = i;
   if( pLoop->wsFlags&WHERE_BTM_LIMIT ){
-    char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName;
+    const char *z = explainIndexColumnName(pIndex, i);
     explainAppendTerm(pStr, i++, z, ">");
   }
   if( pLoop->wsFlags&WHERE_TOP_LIMIT ){
-    char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName;
+    const char *z = explainIndexColumnName(pIndex, j);
     explainAppendTerm(pStr, i, z, "<");
   }
   sqlite3StrAccumAppend(pStr, ")", 1);
@@ -1211,7 +1219,12 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       iRowidReg = ++pParse->nMem;
       sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
       sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
-      sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
+      if( pWInfo->eOnePass!=ONEPASS_OFF ){
+        sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg);
+        VdbeCoverage(v);
+      }else{
+        sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
+      }
     }else if( iCur!=iIdxCur ){
       Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
       iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol);
diff --git a/src/whereexpr.c b/src/whereexpr.c
index 88eb5b70aa..dff425d0ea 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -795,6 +795,51 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
   return mask;
 }
 
+/*
+** Expression pExpr is one operand of a comparison operator that might
+** be useful for indexing.  This routine checks to see if pExpr appears
+** in any index.  Return TRUE (1) if pExpr is an indexed term and return
+** FALSE (0) if not.  If TRUE is returned, also set *piCur to the cursor
+** number of the table that is indexed and *piColumn to the column number
+** of the column that is indexed, or -2 if an expression is being indexed.
+**
+** If pExpr is a TK_COLUMN column reference, then this routine always returns
+** true even if that particular column is not indexed, because the column
+** might be added to an automatic index later.
+*/
+static int exprMightBeIndexed(
+  SrcList *pFrom,        /* The FROM clause */
+  Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
+  Expr *pExpr,           /* An operand of a comparison operator */
+  int *piCur,            /* Write the referenced table cursor number here */
+  int *piColumn          /* Write the referenced table column number here */
+){
+  Index *pIdx;
+  int i;
+  int iCur;
+  if( pExpr->op==TK_COLUMN ){
+    *piCur = pExpr->iTable;
+    *piColumn = pExpr->iColumn;
+    return 1;
+  }
+  if( mPrereq==0 ) return 0;                 /* No table references */
+  if( (mPrereq&(mPrereq-1))!=0 ) return 0;   /* Refs more than one table */
+  for(i=0; mPrereq>1; i++, mPrereq>>=1){}
+  iCur = pFrom->a[i].iCursor;
+  for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    if( pIdx->aColExpr==0 ) continue;
+    for(i=0; inKeyCol; i++){
+      if( pIdx->aiColumn[i]!=(-2) ) continue;
+      if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
+        *piCur = iCur;
+        *piColumn = -2;
+        return 1;
+      }
+    }
+  }
+  return 0;
+}
+
 /*
 ** The input to this routine is an WhereTerm structure with only the
 ** "pExpr" field filled in.  The job of this routine is to analyze the
@@ -865,16 +910,19 @@ static void exprAnalyze(
   pTerm->iParent = -1;
   pTerm->eOperator = 0;
   if( allowedOp(op) ){
+    int iCur, iColumn;
     Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft);
     Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
     u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV;
-    if( pLeft->op==TK_COLUMN ){
-      pTerm->leftCursor = pLeft->iTable;
-      pTerm->u.leftColumn = pLeft->iColumn;
+    if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){
+      pTerm->leftCursor = iCur;
+      pTerm->u.leftColumn = iColumn;
       pTerm->eOperator = operatorMask(op) & opMask;
     }
     if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
-    if( pRight && pRight->op==TK_COLUMN ){
+    if( pRight 
+     && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn)
+    ){
       WhereTerm *pNew;
       Expr *pDup;
       u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
@@ -903,8 +951,8 @@ static void exprAnalyze(
       }
       exprCommute(pParse, pDup);
       pLeft = sqlite3ExprSkipCollate(pDup->pLeft);
-      pNew->leftCursor = pLeft->iTable;
-      pNew->u.leftColumn = pLeft->iColumn;
+      pNew->leftCursor = iCur;
+      pNew->u.leftColumn = iColumn;
       testcase( (prereqLeft | extraRight) != prereqLeft );
       pNew->prereqRight = prereqLeft | extraRight;
       pNew->prereqAll = prereqAll;
diff --git a/test/delete.test b/test/delete.test
index 47d357811b..d2dc106495 100644
--- a/test/delete.test
+++ b/test/delete.test
@@ -68,7 +68,6 @@ do_test delete-3.1.7 {
 } {1 2 4 16}
 integrity_check delete-3.2
 
-
 # Semantic errors in the WHERE clause
 #
 do_test delete-4.1 {
diff --git a/test/delete4.test b/test/delete4.test
new file mode 100644
index 0000000000..7334bf02ed
--- /dev/null
+++ b/test/delete4.test
@@ -0,0 +1,102 @@
+# 2005 August 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is a test of the DELETE command.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix delete4
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t1 VALUES(1, 0);
+  INSERT INTO t1 VALUES(2, 1);
+  INSERT INTO t1 VALUES(3, 0);
+  INSERT INTO t1 VALUES(4, 1);
+  INSERT INTO t1 VALUES(5, 0);
+  INSERT INTO t1 VALUES(6, 1);
+  INSERT INTO t1 VALUES(7, 0);
+  INSERT INTO t1 VALUES(8, 1);
+}
+do_execsql_test 1.2 {
+  DELETE FROM t1 WHERE y=1;
+}
+do_execsql_test 1.3 {
+  SELECT x FROM t1;
+} {1 3 5 7}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.1 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z);
+  INSERT INTO t1 VALUES(1, 0, randomblob(200));
+  INSERT INTO t1 VALUES(2, 1, randomblob(200));
+  INSERT INTO t1 VALUES(3, 0, randomblob(200));
+  INSERT INTO t1 VALUES(4, 1, randomblob(200));
+  INSERT INTO t1 VALUES(5, 0, randomblob(200));
+  INSERT INTO t1 VALUES(6, 1, randomblob(200));
+  INSERT INTO t1 VALUES(7, 0, randomblob(200));
+  INSERT INTO t1 VALUES(8, 1, randomblob(200));
+}
+do_execsql_test 2.2 {
+  DELETE FROM t1 WHERE y=1;
+}
+do_execsql_test 2.3 {
+  SELECT x FROM t1;
+} {1 3 5 7}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.1 {
+  CREATE TABLE t1(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID;
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(2, 4);
+  INSERT INTO t1 VALUES(1, 5);
+  DELETE FROM t1 WHERE a=1;
+  SELECT * FROM t1;
+} {2 4}
+
+#-------------------------------------------------------------------------
+# DELETE statement that uses the OR optimization
+#
+reset_db
+do_execsql_test 3.1 {
+  CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
+  CREATE INDEX i1a ON t1(a);
+  CREATE INDEX i1b ON t1(b);
+  INSERT INTO t1 VALUES(1, 'one', 'i');
+  INSERT INTO t1 VALUES(2, 'two', 'ii');
+  INSERT INTO t1 VALUES(3, 'three', 'iii');
+  INSERT INTO t1 VALUES(4, 'four', 'iv');
+  INSERT INTO t1 VALUES(5, 'one', 'i');
+  INSERT INTO t1 VALUES(6, 'two', 'ii');
+  INSERT INTO t1 VALUES(7, 'three', 'iii');
+  INSERT INTO t1 VALUES(8, 'four', 'iv');
+} {}
+
+do_execsql_test 3.2 {
+  DELETE FROM t1 WHERE a='two' OR b='iv';
+}
+
+do_execsql_test 3.3 {
+  SELECT i FROM t1 ORDER BY i;
+} {1 3 5 7}
+
+do_execsql_test 3.4 { 
+  PRAGMA integrity_check; 
+} {ok}
+
+
+finish_test
diff --git a/test/e_createtable.test b/test/e_createtable.test
index 2921d86c6f..f07fbb9c74 100644
--- a/test/e_createtable.test
+++ b/test/e_createtable.test
@@ -373,8 +373,8 @@ do_createtable_tests 1.1.2 {
 }
 
 
-# EVIDENCE-OF: R-10195-31023 If a  is specified, it
-# must be either "main", "temp", or the name of an attached database.
+# EVIDENCE-OF: R-18448-33677 If a schema-name is specified, it must be
+# either "main", "temp", or the name of an attached database.
 #
 # EVIDENCE-OF: R-39822-07822 In this case the new table is created in
 # the named database.
@@ -422,9 +422,9 @@ do_createtable_tests 1.4 -tclquery {
   2    "CREATE TEMPORARY TABLE t2(a, b)" {{} {t1 t2} {} {}}
 }
 
-# EVIDENCE-OF: R-49439-47561 It is an error to specify both a
-#  and the TEMP or TEMPORARY keyword, unless the
-#  is "temp".
+# EVIDENCE-OF: R-23976-43329 It is an error to specify both a
+# schema-name and the TEMP or TEMPORARY keyword, unless the schema-name
+# is "temp".
 #
 drop_all_tables
 do_createtable_tests 1.5.1 -error {
@@ -447,9 +447,8 @@ do_createtable_tests 1.5.2 -tclquery {
   4    "CREATE TEMPORARY TABLE TEMP.xxx(x)"     {{} {t1 t2 t3 xxx} {} {}}
 }
 
-# EVIDENCE-OF: R-00917-09393 If no database name is specified and the
-# TEMP keyword is not present then the table is created in the main
-# database.
+# EVIDENCE-OF: R-31997-24564 If no schema name is specified and the TEMP
+# keyword is not present then the table is created in the main database.
 #
 drop_all_tables
 do_createtable_tests 1.6 -tclquery {
diff --git a/test/e_delete.test b/test/e_delete.test
index b857cf147f..9bd93229e2 100644
--- a/test/e_delete.test
+++ b/test/e_delete.test
@@ -70,9 +70,12 @@ do_delete_tests e_delete-1.1 {
   2  "DELETE FROM main.t2  ; SELECT * FROM t2"       {}
 }
 
-# EVIDENCE-OF: R-30203-16177 If a WHERE clause is supplied, then only
-# those rows for which the result of evaluating the WHERE clause as a
-# boolean expression is true are deleted.
+# EVIDENCE-OF: R-26300-50198 If a WHERE clause is supplied, then only
+# those rows for which the WHERE clause boolean expression is true are
+# deleted.
+#
+# EVIDENCE-OF: R-23360-48280 Rows for which the expression is false or
+# NULL are retained.
 #
 do_delete_tests e_delete-1.2 {
   1  "DELETE FROM t3 WHERE 1       ; SELECT x FROM t3"       {}
@@ -117,8 +120,8 @@ do_execsql_test e_delete-2.0 {
 # EVIDENCE-OF: R-09681-58560 The table-name specified as part of a
 # DELETE statement within a trigger body must be unqualified.
 #
-# EVIDENCE-OF: R-36771-43788 In other words, the database-name. prefix
-# on the table name is not allowed within triggers.
+# EVIDENCE-OF: R-12275-20298 In other words, the schema-name. prefix on
+# the table name is not allowed within triggers.
 #
 do_delete_tests e_delete-2.1 -error {
   qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers
diff --git a/test/e_expr.test b/test/e_expr.test
index 271635f944..8c0957f8d3 100644
--- a/test/e_expr.test
+++ b/test/e_expr.test
@@ -1426,16 +1426,16 @@ do_execsql_test e_expr-27.1.2 {
     typeof(CAST(4.5 as INTEGER)), CAST(4.5 as INTEGER)
 } {text UVU real 1.23 integer 4}
 
-# EVIDENCE-OF: R-27225-65050 If the value of  is NULL, then
-# the result of the CAST expression is also NULL.
+# EVIDENCE-OF: R-32434-09092 If the value of expr is NULL, then the
+# result of the CAST expression is also NULL.
 #
 do_expr_test e_expr-27.2.1 { CAST(NULL AS integer) } null {}
 do_expr_test e_expr-27.2.2 { CAST(NULL AS text) }    null {}
 do_expr_test e_expr-27.2.3 { CAST(NULL AS blob) }    null {}
 do_expr_test e_expr-27.2.4 { CAST(NULL AS number) }  null {}
 
-# EVIDENCE-OF: R-31076-23575 Casting a value to a  with
-# no affinity causes the value to be converted into a BLOB.
+# EVIDENCE-OF: R-43522-35548 Casting a value to a type-name with no
+# affinity causes the value to be converted into a BLOB.
 #
 do_expr_test e_expr-27.3.1 { CAST('abc' AS blob)       } blob abc
 do_expr_test e_expr-27.3.2 { CAST('def' AS shobblob_x) } blob def
diff --git a/test/e_insert.test b/test/e_insert.test
index ffafa0ea52..32d75cbbd7 100644
--- a/test/e_insert.test
+++ b/test/e_insert.test
@@ -157,9 +157,9 @@ do_insert_tests e_insert-1.1 {
     3b   "SELECT count(*) FROM a2"           {4}
 }
 
-# EVIDENCE-OF: R-53616-44976 If no column-list is specified then the
-# number of values inserted into each row must be the same as the number
-# of columns in the table.
+# EVIDENCE-OF: R-19218-01018 If the column-name list after table-name is
+# omitted then the number of values inserted into each row must be the
+# same as the number of columns in the table.
 #
 #   A test in the block above verifies that if the VALUES list has the
 #   correct number of columns (for table a2, 3 columns) works. So these
@@ -191,9 +191,9 @@ do_insert_tests e_insert-1.3 {
     3b   "SELECT * FROM a2 WHERE oid=last_insert_rowid()" {2 x y}
 }
 
-# EVIDENCE-OF: R-09234-17933 If a column-list is specified, then the
-# number of values in each term of the VALUE list must match the number
-# of specified columns.
+# EVIDENCE-OF: R-21115-58321 If a column-name list is specified, then
+# the number of values in each term of the VALUE list must match the
+# number of specified columns.
 #
 do_insert_tests e_insert-1.4 -error { 
   %d values for %d columns
@@ -394,8 +394,8 @@ foreach {tn sql error ac data } {
   do_test          e_insert-4.1.$tn.3 {sqlite3_get_autocommit db} $ac
 }
 
-# EVIDENCE-OF: R-64196-02418 The optional "database-name." prefix on the
-# table-name is support for top-level INSERT statements only.
+# EVIDENCE-OF: R-59829-49719 The optional "schema-name." prefix on the
+# table-name is supported for top-level INSERT statements only.
 #
 # EVIDENCE-OF: R-05731-00924 The table name must be unqualified for
 # INSERT statements that occur within CREATE TRIGGER statements.
diff --git a/test/e_reindex.test b/test/e_reindex.test
index fa66aa7a1f..c6a9e0352f 100644
--- a/test/e_reindex.test
+++ b/test/e_reindex.test
@@ -265,10 +265,9 @@ test_index 5.40  t2     collB   value
 test_index 5.41  aux.t1 collA   length
 test_index 5.42  aux.t1 collB   value
 
-# EVIDENCE-OF: R-15639-02023 If no database-name is specified and there
-# exists both a table or index and a collation sequence of the specified
-# name, SQLite interprets this as a request to rebuild the indices that
-# use the named collation sequence.
+# EVIDENCE-OF: R-35892-30289 For a command of the form "REINDEX name", a
+# match against collation-name takes precedence over a match against
+# index-name or table-name.
 #
 set_collations value length
 do_execsql_test e_reindex-2.6.0 {
diff --git a/test/e_resolve.test b/test/e_resolve.test
index 512fcf2748..f4bb1a4317 100644
--- a/test/e_resolve.test
+++ b/test/e_resolve.test
@@ -65,9 +65,9 @@ do_execsql_test 1.2 { SELECT * FROM n2 } {main n2}
 do_execsql_test 1.3 { SELECT * FROM n3 } {at1  n3}
 do_execsql_test 1.4 { SELECT * FROM n4 } {at2  n4}
 
-# EVIDENCE-OF: R-54577-28142 If a database name is specified as part of
-# an object reference, it must be either "main", or "temp" or the name
-# of an attached database.
+# EVIDENCE-OF: R-00634-08585 If a schema name is specified as part of an
+# object reference, it must be either "main", or "temp" or the
+# schema-name of an attached database.
 #
 #   Or else it is a "no such table: xxx" error.
 #
@@ -79,7 +79,7 @@ do_execsql_test 2.1.4 { SELECT * FROM at2.n1 } {at2 n1}
 
 do_catchsql_test 2.2 { SELECT * FROM xxx.n1 } {1 {no such table: xxx.n1}}
 
-# EVIDENCE-OF: R-26223-47623 Like other SQL identifiers, database names
+# EVIDENCE-OF: R-17446-42210 Like other SQL identifiers, schema names
 # are case-insensitive.
 #
 resolve_reopen_db
@@ -88,8 +88,8 @@ do_execsql_test 3.2 { SELECT * FROM tEmP.n1 } {temp n1}
 do_execsql_test 3.3 { SELECT * FROM aT1.n1 } {at1 n1}
 do_execsql_test 3.4 { SELECT * FROM At2.n1 } {at2 n1}
 
-# EVIDENCE-OF: R-15639-28392 If a database name is specified, then only
-# the named database is searched for the named object.
+# EVIDENCE-OF: R-14755-58619 If a schema name is specified, then only
+# that one schema is searched for the named object.
 #
 do_catchsql_test 4.1 { SELECT * FROM temp.n2 } {1 {no such table: temp.n2}}
 do_catchsql_test 4.2 { SELECT * FROM main.n2 } {0 {main n2}}
diff --git a/test/e_update.test b/test/e_update.test
index e9c6b26809..a13b059b32 100644
--- a/test/e_update.test
+++ b/test/e_update.test
@@ -146,9 +146,8 @@ do_update_tests e_update-1.2 {
      {greek roman  greek roman  greek roman}
 }
 
-# EVIDENCE-OF: R-42117-40023 Otherwise, the UPDATE affects only those
-# rows for which the result of evaluating the WHERE clause expression as
-# a boolean expression is true.
+# EVIDENCE-OF: R-58095-46013 Otherwise, the UPDATE affects only those
+# rows for which the WHERE clause boolean expression is true.
 #
 do_execsql_test e_update-1.3.0 {
   DELETE FROM main.t1;
@@ -265,9 +264,10 @@ do_update_tests e_update-1.7 -query {
   3   "UPDATE t2 SET a=c||c, c=NULL" {44 5 {}  99 14 {}  55 11 {}}
 }
 
-# EVIDENCE-OF: R-12619-24112 The optional conflict-clause allows the
-# user to nominate a specific constraint conflict resolution algorithm
-# to use during this one UPDATE command.
+# EVIDENCE-OF: R-28518-13457 The optional "OR action" conflict clause
+# that follows the UPDATE keyword allows the user to nominate a specific
+# constraint conflict resolution algorithm to use during this one UPDATE
+# command.
 #
 do_execsql_test e_update-1.8.0 {
   DELETE FROM t3;
@@ -322,8 +322,8 @@ foreach {tn sql error ac data } {
 # EVIDENCE-OF: R-12123-54095 The table-name specified as part of an
 # UPDATE statement within a trigger body must be unqualified.
 #
-# EVIDENCE-OF: R-09690-36749 In other words, the database-name. prefix
-# on the table name of the UPDATE is not allowed within triggers.
+# EVIDENCE-OF: R-43190-62442 In other words, the schema-name. prefix on
+# the table name of the UPDATE is not allowed within triggers.
 #
 do_update_tests e_update-2.1 -error {
   qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers
diff --git a/test/fkey5.test b/test/fkey5.test
index 21feb2bb8b..b9e1fc2eec 100644
--- a/test/fkey5.test
+++ b/test/fkey5.test
@@ -12,8 +12,8 @@
 #
 # This file tests the PRAGMA foreign_key_check command.
 #
-# EVIDENCE-OF: R-01427-50262 PRAGMA database.foreign_key_check; PRAGMA
-# database.foreign_key_check(table-name);
+# EVIDENCE-OF: R-15402-03103 PRAGMA schema.foreign_key_check; PRAGMA
+# schema.foreign_key_check(table-name);
 #
 # EVIDENCE-OF: R-23918-17301 The foreign_key_check pragma checks the
 # database, or the table called "table-name", for foreign key
diff --git a/test/index.test b/test/index.test
index 59c0ea6f90..712f42c3a4 100644
--- a/test/index.test
+++ b/test/index.test
@@ -56,11 +56,11 @@ do_test index-2.1 {
 # Try adding an index on a column of a table where the table
 # exists but the column does not.
 #
-do_test index-2.1 {
+do_test index-2.1b {
   execsql {CREATE TABLE test1(f1 int, f2 int, f3 int)}
   set v [catch {execsql {CREATE INDEX index1 ON test1(f4)}} msg]
   lappend v $msg
-} {1 {table test1 has no column named f4}}
+} {1 {no such column: f4}}
 
 # Try an index with some columns that match and others that do now.
 #
@@ -68,7 +68,7 @@ do_test index-2.2 {
   set v [catch {execsql {CREATE INDEX index1 ON test1(f1, f2, f4, f3)}} msg]
   execsql {DROP TABLE test1}
   lappend v $msg
-} {1 {table test1 has no column named f4}}
+} {1 {no such column: f4}}
 
 # Try creating a bunch of indices on the same table
 #
diff --git a/test/index2.test b/test/index2.test
index 48d0c38e18..b1d7e227fb 100644
--- a/test/index2.test
+++ b/test/index2.test
@@ -1,4 +1,4 @@
-# 2005 January 11
+# 2005-01-11
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
@@ -11,7 +11,6 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the CREATE INDEX statement.
 #
-# $Id: index2.test,v 1.3 2006/03/03 19:12:30 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
diff --git a/test/index3.test b/test/index3.test
index 0cdc6e088d..1d90de1b9b 100644
--- a/test/index3.test
+++ b/test/index3.test
@@ -1,4 +1,4 @@
-# 2005 February 14
+# 2005-02-14
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
@@ -11,7 +11,6 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the CREATE INDEX statement.
 #
-# $Id: index3.test,v 1.3 2008/03/19 13:03:34 drh Exp $
 
 
 set testdir [file dirname $argv0]
@@ -40,17 +39,42 @@ do_test index3-1.3 {
 } {0 {}}
 integrity_check index3-1.4
 
+# Backwards compatibility test:
+#
+# Verify that CREATE INDEX statements that use strings instead of 
+# identifiers for the the column names continue to work correctly.
+# This is undocumented behavior retained for backwards compatiblity.
+#
+do_execsql_test index3-2.1 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a, b, c, d, e, 
+                  PRIMARY KEY('a'), UNIQUE('b' COLLATE nocase DESC));
+  CREATE INDEX t1c ON t1('c');
+  CREATE INDEX t1d ON t1('d' COLLATE binary ASC);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<30)
+    INSERT INTO t1(a,b,c,d,e) 
+      SELECT x, printf('ab%03xxy',x), x, x, x FROM c;
+} {}
+do_execsql_test index3-2.2 {
+  SELECT a FROM t1 WHERE b='ab005xy' COLLATE nocase;
+} {5}
+do_execsql_test index3-2.2eqp {
+  EXPLAIN QUERY PLAN
+  SELECT a FROM t1 WHERE b='ab005xy' COLLATE nocase;
+} {/USING INDEX/}
+
+
 # This test corrupts the database file so it must be the last test
 # in the series.
 #
 do_test index3-99.1 {
   execsql {
     PRAGMA writable_schema=on;
-    UPDATE sqlite_master SET sql='nonsense';
+    UPDATE sqlite_master SET sql='nonsense' WHERE name='t1d'
   }
   db close
   catch { sqlite3 db test.db }
-  catchsql { DROP INDEX i1 }
-} {1 {malformed database schema (t1)}}
+  catchsql { DROP INDEX t1c }
+} {1 {malformed database schema (t1d)}}
 
 finish_test
diff --git a/test/indexedby.test b/test/indexedby.test
index 413bf07422..83c7a5cccc 100644
--- a/test/indexedby.test
+++ b/test/indexedby.test
@@ -58,7 +58,7 @@ do_execsql_test indexedby-1.4 {
 # SQL view. Also test that specifying an index that does not exist or
 # is attached to a different table is detected as an error.
 #
-# EVIDENCE-OF: R-63761-48810 -- syntax diagram qualified-table-name
+# EVIDENCE-OF: R-07004-11522 -- syntax diagram qualified-table-name
 # 
 # EVIDENCE-OF: R-58230-57098 The "INDEXED BY index-name" phrase
 # specifies that the named index must be used in order to look up values
@@ -231,13 +231,13 @@ do_execsql_test indexedby-6.2 {
 # 
 do_execsql_test indexedby-7.1 {
   EXPLAIN QUERY PLAN DELETE FROM t1 WHERE a = 5 
-} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}}
 do_execsql_test indexedby-7.2 {
   EXPLAIN QUERY PLAN DELETE FROM t1 NOT INDEXED WHERE a = 5 
 } {0 0 0 {SCAN TABLE t1}}
 do_execsql_test indexedby-7.3 {
   EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i1 WHERE a = 5 
-} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}}
 do_execsql_test indexedby-7.4 {
   EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i1 WHERE a = 5 AND b = 10
 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}}
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
new file mode 100644
index 0000000000..0c925c9f92
--- /dev/null
+++ b/test/indexexpr1.test
@@ -0,0 +1,222 @@
+# 2015-08-31
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing indexes on expressions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test indexexpr1-100 {
+  CREATE TABLE t1(a,b,c);
+  INSERT INTO t1(a,b,c)
+      /*  123456789 123456789 123456789 123456789 123456789 123456789 */ 
+  VALUES('In_the_beginning_was_the_Word',1,1),
+        ('and_the_Word_was_with_God',1,2),
+        ('and_the_Word_was_God',1,3),
+        ('The_same_was_in_the_beginning_with_God',2,1),
+        ('All_things_were_made_by_him',3,1),
+        ('and_without_him_was_not_any_thing_made_that_was_made',3,2);
+  CREATE INDEX t1a1 ON t1(substr(a,1,12));
+} {}
+do_execsql_test indexexpr1-110 {
+  SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c;
+} {1 2 | 1 3 |}
+do_execsql_test indexexpr1-110eqp {
+  EXPLAIN QUERY PLAN
+  SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c;
+} {/USING INDEX t1a1/}
+do_execsql_test indexexpr1-120 {
+  SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c;
+} {1 2 | 1 3 |}
+do_execsql_test indexexpr1-120eqp {
+  EXPLAIN QUERY PLAN
+  SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c;
+} {/USING INDEX t1a1/}
+
+do_execsql_test indexexpr1-130 {
+  CREATE INDEX t1ba ON t1(b,substr(a,2,3),c);
+  SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c;
+} {2 3}
+do_execsql_test indexexpr1-130eqp {
+  EXPLAIN QUERY PLAN
+  SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c;
+} {/USING INDEX t1ba/}
+
+do_execsql_test indexexpr1-140 {
+  SELECT rowid, substr(a,b,3), '|' FROM t1 ORDER BY 2;
+} {1 In_ | 2 and | 3 and | 6 d_w | 4 he_ | 5 l_t |}
+do_execsql_test indexexpr1-141 {
+  CREATE INDEX t1abx ON t1(substr(a,b,3));
+  SELECT rowid FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +rowid;
+} {1 2 3}
+do_execsql_test indexexpr1-141eqp {
+  EXPLAIN QUERY PLAN
+  SELECT rowid FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +rowid;
+} {/USING INDEX t1abx/}
+do_execsql_test indexexpr1-142 {
+  SELECT rowid FROM t1 WHERE +substr(a,b,3)<='and' ORDER BY +rowid;
+} {1 2 3}
+do_execsql_test indexexpr1-150 {
+  SELECT rowid FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz')
+   ORDER BY +rowid;
+} {2 3 5}
+do_execsql_test indexexpr1-150eqp {
+  EXPLAIN QUERY PLAN
+  SELECT rowid FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz')
+   ORDER BY +rowid;
+} {/USING INDEX t1abx/}
+
+do_execsql_test indexexpr1-160 {
+  ALTER TABLE t1 ADD COLUMN d;
+  UPDATE t1 SET d=length(a);
+  CREATE INDEX t1a2 ON t1(SUBSTR(a, 27, 3)) WHERE d>=29;
+  SELECT rowid, b, c FROM t1
+   WHERE substr(a,27,3)=='ord' AND d>=29;
+} {1 1 1}
+do_execsql_test indexexpr1-160eqp {
+  EXPLAIN QUERY PLAN
+  SELECT rowid, b, c FROM t1
+   WHERE substr(a,27,3)=='ord' AND d>=29;
+} {/USING INDEX t1a2/}
+
+
+do_execsql_test indexexpr1-200 {
+  DROP TABLE t1;
+  CREATE TABLE t1(id ANY PRIMARY KEY, a,b,c) WITHOUT ROWID;
+  INSERT INTO t1(id,a,b,c)
+  VALUES(1,'In_the_beginning_was_the_Word',1,1),
+        (2,'and_the_Word_was_with_God',1,2),
+        (3,'and_the_Word_was_God',1,3),
+        (4,'The_same_was_in_the_beginning_with_God',2,1),
+        (5,'All_things_were_made_by_him',3,1),
+        (6,'and_without_him_was_not_any_thing_made_that_was_made',3,2);
+  CREATE INDEX t1a1 ON t1(substr(a,1,12));
+} {}
+do_execsql_test indexexpr1-210 {
+  SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c;
+} {1 2 | 1 3 |}
+do_execsql_test indexexpr1-210eqp {
+  EXPLAIN QUERY PLAN
+  SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c;
+} {/USING INDEX t1a1/}
+do_execsql_test indexexpr1-220 {
+  SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c;
+} {1 2 | 1 3 |}
+do_execsql_test indexexpr1-220eqp {
+  EXPLAIN QUERY PLAN
+  SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c;
+} {/USING INDEX t1a1/}
+
+do_execsql_test indexexpr1-230 {
+  CREATE INDEX t1ba ON t1(b,substr(a,2,3),c);
+  SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c;
+} {2 3}
+do_execsql_test indexexpr1-230eqp {
+  EXPLAIN QUERY PLAN
+  SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c;
+} {/USING INDEX t1ba/}
+
+do_execsql_test indexexpr1-240 {
+  SELECT id, substr(a,b,3), '|' FROM t1 ORDER BY 2;
+} {1 In_ | 2 and | 3 and | 6 d_w | 4 he_ | 5 l_t |}
+do_execsql_test indexexpr1-241 {
+  CREATE INDEX t1abx ON t1(substr(a,b,3));
+  SELECT id FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +id;
+} {1 2 3}
+do_execsql_test indexexpr1-241eqp {
+  EXPLAIN QUERY PLAN
+  SELECT id FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +id;
+} {/USING INDEX t1abx/}
+do_execsql_test indexexpr1-242 {
+  SELECT id FROM t1 WHERE +substr(a,b,3)<='and' ORDER BY +id;
+} {1 2 3}
+do_execsql_test indexexpr1-250 {
+  SELECT id FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz')
+   ORDER BY +id;
+} {2 3 5}
+do_execsql_test indexexpr1-250eqp {
+  EXPLAIN QUERY PLAN
+  SELECT id FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz')
+   ORDER BY +id;
+} {/USING INDEX t1abx/}
+
+do_execsql_test indexexpr1-260 {
+  ALTER TABLE t1 ADD COLUMN d;
+  UPDATE t1 SET d=length(a);
+  CREATE INDEX t1a2 ON t1(SUBSTR(a, 27, 3)) WHERE d>=29;
+  SELECT id, b, c FROM t1
+   WHERE substr(a,27,3)=='ord' AND d>=29;
+} {1 1 1}
+do_execsql_test indexexpr1-260eqp {
+  EXPLAIN QUERY PLAN
+  SELECT id, b, c FROM t1
+   WHERE substr(a,27,3)=='ord' AND d>=29;
+} {/USING INDEX t1a2/}
+
+
+do_catchsql_test indexexpr1-300 {
+  CREATE TABLE t2(a,b,c);
+  CREATE INDEX t2x1 ON t2(a,b+random());
+} {1 {non-deterministic functions prohibited in index expressions}}
+do_catchsql_test indexexpr1-301 {
+  CREATE INDEX t2x1 ON t2(a+julianday('now'));
+} {1 {non-deterministic functions prohibited in index expressions}}
+do_catchsql_test indexexpr1-310 {
+  CREATE INDEX t2x2 ON t2(a,b+(SELECT 15));
+} {1 {subqueries prohibited in index expressions}}
+do_catchsql_test indexexpr1-320 {
+  CREATE TABLE e1(x,y,UNIQUE(y,substr(x,1,5)));
+} {1 {expressions prohibited in PRIMARY KEY and UNIQUE constraints}}
+do_catchsql_test indexexpr1-330 {
+  CREATE TABLE e1(x,y,PRIMARY KEY(y,substr(x,1,5)));
+} {1 {expressions prohibited in PRIMARY KEY and UNIQUE constraints}}
+do_catchsql_test indexexpr1-331 {
+  CREATE TABLE e1(x,y,PRIMARY KEY(y,substr(x,1,5))) WITHOUT ROWID;
+} {1 {expressions prohibited in PRIMARY KEY and UNIQUE constraints}}
+do_catchsql_test indexexpr1-340 {
+  CREATE TABLE e1(x,y,FOREIGN KEY(substr(y,1,5)) REFERENCES t1);
+} {1 {near "(": syntax error}}
+
+do_execsql_test indexexpr1-400 {
+  CREATE TABLE t3(a,b,c);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<30)
+  INSERT INTO t3(a,b,c)
+    SELECT x, printf('ab%04xyz',x), random() FROM c;
+  CREATE UNIQUE INDEX t3abc ON t3(CAST(a AS text), b, substr(c,1,3));
+  SELECT a FROM t3 WHERE CAST(a AS text)<='10' ORDER BY +a;
+} {1 10}
+do_catchsql_test indexexpr1-410 {
+  INSERT INTO t3 SELECT * FROM t3 WHERE rowid=10;
+} {1 {UNIQUE constraint failed: index 't3abc'}}
+
+do_execsql_test indexexpr1-500 {
+  CREATE TABLE t5(a);
+  CREATE TABLE cnt(x);
+  WITH RECURSIVE
+    c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<5)
+  INSERT INTO cnt(x) SELECT x FROM c;
+  INSERT INTO t5(a) SELECT printf('abc%03dxyz',x) FROM cnt;
+  CREATE INDEX t5ax ON t5( substr(a,4,3) );
+} {}
+do_execsql_test indexexpr1-510 {
+  -- The use of the "k" alias in the WHERE clause is technically
+  -- illegal, but SQLite allows it for historical reasons.  In this
+  -- test and the next, verify that "k" can be used by the t5ax index
+  SELECT substr(a,4,3) AS k FROM cnt, t5 WHERE k=printf('%03d',x);
+} {001 002 003 004 005}
+do_execsql_test indexexpr1-510eqp {
+  EXPLAIN QUERY PLAN
+  SELECT substr(a,4,3) AS k FROM cnt, t5 WHERE k=printf('%03d',x);
+} {/USING INDEX t5ax/}
+
+
+finish_test
diff --git a/test/json101.test b/test/json101.test
index 1a84a5fc5e..9543ccfd24 100644
--- a/test/json101.test
+++ b/test/json101.test
@@ -16,9 +16,21 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
 load_static_extension db json
-do_execsql_test json1-1.1 {
+do_execsql_test json1-1.1.00 {
   SELECT json_array(1,2.5,null,'hello');
 } {[1,2.5,null,"hello"]}
+do_execsql_test json1-1.1.01 {
+  SELECT json_array(1,'{"abc":2.5,"def":null,"ghi":hello}',99);
+  -- the second term goes in as a string:
+} {[1,"{\\"abc\\":2.5,\\"def\\":null,\\"ghi\\":hello}",99]}
+do_execsql_test json1-1.1.02 {
+  SELECT json_array(1,json('{"abc":2.5,"def":null,"ghi":"hello"}'),99);
+  -- the second term goes in as JSON
+} {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]}
+do_execsql_test json1-1.1.03 {
+  SELECT json_array(1,json_object('abc',2.5,'def',null,'ghi','hello'),99);
+  -- the second term goes in as JSON
+} {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]}
 do_execsql_test json1-1.2 {
   SELECT hex(json_array('String "\ Test'));
 } {5B22537472696E67205C225C5C2054657374225D}
@@ -54,13 +66,13 @@ do_execsql_test json1-3.1 {
   SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]');
 } {{{"a":"[3,4,5]","b":2}}}
 do_execsql_test json1-3.2 {
-  SELECT json_replace('{"a":1,"b":2}','$$.a','[3,4,5]');
+  SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]'));
 } {{{"a":[3,4,5],"b":2}}}
 do_execsql_test json1-3.3 {
   SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b');
 } {text}
 do_execsql_test json1-3.4 {
-  SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b');
+  SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b');
 } {object}
 
 # Per rfc7159, any JSON value is allowed at the top level, and whitespace
diff --git a/test/json102.test b/test/json102.test
new file mode 100644
index 0000000000..da9fbd1b76
--- /dev/null
+++ b/test/json102.test
@@ -0,0 +1,281 @@
+# 2015-08-12
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for JSON SQL functions extension to the
+# SQLite library.
+#
+# This file contains tests automatically generated from the json1
+# documentation.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+load_static_extension db json
+do_execsql_test json102-100 {
+  SELECT json_object('ex','[52,3.14159]');
+} {{{"ex":"[52,3.14159]"}}}
+do_execsql_test json102-110 {
+  SELECT json_object('ex',json('[52,3.14159]'));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120 {
+  SELECT json_object('ex',json_array(52,3.14159));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-130 {
+  SELECT json(' { "this" : "is", "a": [ "test" ] } ');
+} {{{"this":"is","a":["test"]}}}
+do_execsql_test json102-140 {
+  SELECT json_array(1,2,'3',4);
+} {{[1,2,"3",4]}}
+do_execsql_test json102-150 {
+  SELECT json_array('[1,2]');
+} {{["[1,2]"]}}
+do_execsql_test json102-160 {
+  SELECT json_array(json_array(1,2));
+} {{[[1,2]]}}
+do_execsql_test json102-170 {
+  SELECT json_array(1,null,'3','[4,5]','{"six":7.7}');
+} {{[1,null,"3","[4,5]","{\"six\":7.7}"]}}
+do_execsql_test json102-180 {
+  SELECT json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}'));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-190 {
+  SELECT json_array_length('[1,2,3,4]');
+} {{4}}
+do_execsql_test json102-200 {
+  SELECT json_array_length('[1,2,3,4]', '$');
+} {{4}}
+do_execsql_test json102-210 {
+  SELECT json_array_length('[1,2,3,4]', '$[2]');
+} {{0}}
+do_execsql_test json102-220 {
+  SELECT json_array_length('{"one":[1,2,3]}');
+} {{0}}
+do_execsql_test json102-230 {
+  SELECT json_array_length('{"one":[1,2,3]}', '$.one');
+} {{3}}
+do_execsql_test json102-240 {
+  SELECT json_array_length('{"one":[1,2,3]}', '$.two');
+} {{}}
+do_execsql_test json102-250 {
+  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$');
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-260 {
+  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c');
+} {{[4,5,{"f":7}]}}
+do_execsql_test json102-270 {
+  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]');
+} {{{"f":7}}}
+do_execsql_test json102-280 {
+  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f');
+} {{7}}
+do_execsql_test json102-290 {
+  SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a');
+} {{[[4,5],2]}}
+do_execsql_test json102-300 {
+  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x');
+} {{}}
+do_execsql_test json102-310 {
+  SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a');
+} {{[null,2]}}
+do_execsql_test json102-320 {
+  SELECT json_insert('{"a":2,"c":4}', '$.a', 99);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-330 {
+  SELECT json_insert('{"a":2,"c":4}', '$.e', 99);
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-340 {
+  SELECT json_replace('{"a":2,"c":4}', '$.a', 99);
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-350 {
+  SELECT json_replace('{"a":2,"c":4}', '$.e', 99);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-360 {
+  SELECT json_set('{"a":2,"c":4}', '$.a', 99);
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-370 {
+  SELECT json_set('{"a":2,"c":4}', '$.e', 99);
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-380 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', '[97,96]');
+} {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-390 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', json('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', json_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-410 {
+  SELECT json_object('a',2,'c',4);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-420 {
+  SELECT json_object('a',2,'c','{e:5}');
+} {{{"a":2,"c":"{e:5}"}}}
+do_execsql_test json102-430 {
+  SELECT json_object('a',2,'c',json_object('e',5));
+} {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-440 {
+  SELECT json_remove('[0,1,2,3,4]','$[2]');
+} {{[0,1,3,4]}}
+do_execsql_test json102-450 {
+  SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]');
+} {{[1,3,4]}}
+do_execsql_test json102-460 {
+  SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]');
+} {{[1,2,4]}}
+do_execsql_test json102-470 {
+  SELECT json_remove('{"x":25,"y":42}');
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-480 {
+  SELECT json_remove('{"x":25,"y":42}','$.z');
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-490 {
+  SELECT json_remove('{"x":25,"y":42}','$.y');
+} {{{"x":25}}}
+do_execsql_test json102-500 {
+  SELECT json_remove('{"x":25,"y":42}','$');
+} {{}}
+do_execsql_test json102-510 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}');
+} {{object}}
+do_execsql_test json102-520 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$');
+} {{object}}
+do_execsql_test json102-530 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a');
+} {{array}}
+do_execsql_test json102-540 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]');
+} {{integer}}
+do_execsql_test json102-550 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]');
+} {{real}}
+do_execsql_test json102-560 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]');
+} {{true}}
+do_execsql_test json102-570 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]');
+} {{false}}
+do_execsql_test json102-580 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]');
+} {{null}}
+do_execsql_test json102-590 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]');
+} {{text}}
+do_execsql_test json102-600 {
+  SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]');
+} {{}}
+do_execsql_test json102-610 {
+  SELECT json_valid(char(123)||'"x":35'||char(125));
+} {{1}}
+do_execsql_test json102-620 {
+  SELECT json_valid(char(123)||'"x":35');
+} {{0}}
+
+ifcapable vtab {
+do_execsql_test json102-1000 {
+  CREATE TABLE user(name,phone);
+  INSERT INTO user(name,phone) VALUES
+     ('Alice','["919-555-2345","804-555-3621"]'),
+     ('Bob','["201-555-8872"]'),
+     ('Cindy','["704-555-9983"]'),
+     ('Dave','["336-555-8421","704-555-4321","803-911-4421"]');
+  SELECT DISTINCT user.name
+    FROM user, json_each(user.phone)
+   WHERE json_each.value LIKE '704-%'
+   ORDER BY 1;
+} {Cindy Dave}
+
+do_execsql_test json102-1010 {
+  UPDATE user
+     SET phone=json_extract(phone,'$[0]')
+   WHERE json_array_length(phone)<2;
+  SELECT name, substr(phone,1,5) FROM user ORDER BY name;
+} {Alice {["919} Bob 201-5 Cindy 704-5 Dave {["336}}
+do_execsql_test json102-1011 {
+  SELECT name FROM user WHERE phone LIKE '704-%'
+  UNION
+  SELECT user.name
+    FROM user, json_each(user.phone)
+   WHERE json_valid(user.phone)
+     AND json_each.value LIKE '704-%';
+} {Cindy Dave}
+
+do_execsql_test json102-1100 {
+  CREATE TABLE big(json JSON);
+  INSERT INTO big(json) VALUES('{
+    "id":123,
+    "stuff":[1,2,3,4],
+    "partlist":[
+       {"uuid":"bb108722-572e-11e5-9320-7f3b63a4ca74"},
+       {"uuid":"c690dc14-572e-11e5-95f9-dfc8861fd535"},
+       {"subassembly":[
+          {"uuid":"6fa5181e-5721-11e5-a04e-57f3d7b32808"}
+       ]}
+    ]
+  }');
+  INSERT INTO big(json) VALUES('{
+    "id":456,
+    "stuff":["hello","world","xyzzy"],
+    "partlist":[
+       {"uuid":false},
+       {"uuid":"c690dc14-572e-11e5-95f9-dfc8861fd535"}
+    ]
+  }');
+} {}
+set correct_answer [list \
+    1 {$.id} 123 \
+    1 {$.stuff[0]} 1 \
+    1 {$.stuff[1]} 2 \
+    1 {$.stuff[2]} 3 \
+    1 {$.stuff[3]} 4 \
+    1 {$.partlist[0].uuid} bb108722-572e-11e5-9320-7f3b63a4ca74 \
+    1 {$.partlist[1].uuid} c690dc14-572e-11e5-95f9-dfc8861fd535 \
+    1 {$.partlist[2].subassembly[0].uuid} 6fa5181e-5721-11e5-a04e-57f3d7b32808 \
+    2 {$.id} 456 \
+    2 {$.stuff[0]} hello \
+    2 {$.stuff[1]} world \
+    2 {$.stuff[2]} xyzzy \
+    2 {$.partlist[0].uuid} 0 \
+    2 {$.partlist[1].uuid} c690dc14-572e-11e5-95f9-dfc8861fd535]
+do_execsql_test json102-1110 {
+  SELECT big.rowid, fullkey, value
+    FROM big, json_tree(big.json)
+   WHERE json_tree.type NOT IN ('object','array')
+   ORDER BY +big.rowid, +json_tree.id
+} $correct_answer
+do_execsql_test json102-1120 {
+  SELECT big.rowid, fullkey, atom
+    FROM big, json_tree(big.json)
+   WHERE atom IS NOT NULL
+   ORDER BY +big.rowid, +json_tree.id
+} $correct_answer
+
+do_execsql_test json102-1130 {
+  SELECT DISTINCT json_extract(big.json,'$.id')
+    FROM big, json_tree(big.json,'$.partlist')
+   WHERE json_tree.key='uuid'
+     AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808';
+} {123}
+do_execsql_test json102-1131 {
+  SELECT DISTINCT json_extract(big.json,'$.id')
+    FROM big, json_tree(big.json,'$')
+   WHERE json_tree.key='uuid'
+     AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808';
+} {123}
+do_execsql_test json102-1132 {
+  SELECT DISTINCT json_extract(big.json,'$.id')
+    FROM big, json_tree(big.json)
+   WHERE json_tree.key='uuid'
+     AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808';
+} {123}
+} ;# end ifcapable vtab
+
+finish_test
diff --git a/test/pragma.test b/test/pragma.test
index 587a03c8a6..246a7cd854 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -83,8 +83,8 @@ delete_file test.db test.db-journal
 delete_file test3.db test3.db-journal
 sqlite3 db test.db; set DB [sqlite3_connection_pointer db]
 
-# EVIDENCE-OF: R-24197-42751 PRAGMA database.cache_size; PRAGMA
-# database.cache_size = pages; PRAGMA database.cache_size = -kibibytes;
+# EVIDENCE-OF: R-13861-56665 PRAGMA schema.cache_size; PRAGMA
+# schema.cache_size = pages; PRAGMA schema.cache_size = -kibibytes;
 # Query or change the suggested maximum number of database disk pages
 # that SQLite will hold in memory at once per open database file.
 #
@@ -697,8 +697,8 @@ do_test pragma-6.5.1b {
 } {0 0 a 1 1 b 2 -1 {}}
 
 
-# EVIDENCE-OF: R-62725-03366 PRAGMA database.index_info(index-name);
-# This pragma returns one row for each key column in the named index.
+# EVIDENCE-OF: R-29448-60346 PRAGMA schema.index_info(index-name); This
+# pragma returns one row for each key column in the named index.
 #
 # (The first column of output from PRAGMA index_info is...)
 # EVIDENCE-OF: R-34186-52914 The rank of the column within the index. (0
@@ -784,9 +784,9 @@ do_test pragma-6.8 {
 # Miscellaneous tests
 #
 ifcapable schema_pragmas {
-# EVIDENCE-OF: R-63500-32024 PRAGMA database.index_list(table-name);
-# This pragma returns one row for each index associated with the given
-# table.
+# EVIDENCE-OF: R-64103-17776 PRAGMA schema.index_list(table-name); This
+# pragma returns one row for each index associated with the given table.
+#
 do_test pragma-7.1.1 {
   # Make sure a pragma knows to read the schema if it needs to
   db close
@@ -1379,8 +1379,8 @@ ifcapable pager_pragmas {
   forcedelete test.db
   sqlite3 db test.db
  
-  # EVIDENCE-OF: R-13905-26312 PRAGMA database.page_count; Return the
-  # total number of pages in the database file.
+  # EVIDENCE-OF: R-15672-33611 PRAGMA schema.page_count; Return the total
+  # number of pages in the database file.
   #
   do_test pragma-14.1 {
     execsql { pragma auto_vacuum = 0 }
@@ -1817,8 +1817,8 @@ do_test 23.2a {
   db2 eval {SELECT cid, name, '|' FROM out ORDER BY seqno}
 } {2 c | 3 d | 1 b |}
 
-# EVIDENCE-OF: R-44874-46325 PRAGMA database.index_xinfo(index-name);
-# This pragma returns information about every column in an index.
+# EVIDENCE-OF: R-56143-29319 PRAGMA schema.index_xinfo(index-name); This
+# pragma returns information about every column in an index.
 #
 # EVIDENCE-OF: R-45970-35618 Unlike this index_info pragma, this pragma
 # returns information about every column in the index, not just the key
@@ -1861,9 +1861,8 @@ do_test 23.2d {
   db2 eval {PRAGMA index_xinfo(i2x)}
 } {0 3 d 0 nocase 1 1 2 c 1 BINARY 1 2 -1 {} 0 BINARY 0}
 
-# EVIDENCE-OF: R-63500-32024 PRAGMA database.index_list(table-name);
-# This pragma returns one row for each index associated with the given
-# table.
+# EVIDENCE-OF: R-64103-17776 PRAGMA schema.index_list(table-name); This
+# pragma returns one row for each index associated with the given table.
 #
 # (The first column of output from PRAGMA index_list is...)
 # EVIDENCE-OF: R-02753-24748 A sequence number assigned to each index
diff --git a/test/pragma2.test b/test/pragma2.test
index 12d5cce5f7..e2c87019eb 100644
--- a/test/pragma2.test
+++ b/test/pragma2.test
@@ -42,7 +42,7 @@ sqlite3 db test.db; set DB [sqlite3_connection_pointer db]
 db eval {PRAGMA auto_vacuum=0}
 
 
-# EVIDENCE-OF: R-17887-14874 PRAGMA database.freelist_count; Return the
+# EVIDENCE-OF: R-11211-21323 PRAGMA schema.freelist_count; Return the
 # number of unused pages in the database file.
 #
 do_test pragma2-1.1 {
diff --git a/test/releasetest.tcl b/test/releasetest.tcl
index eadf69ce81..55acd215e7 100644
--- a/test/releasetest.tcl
+++ b/test/releasetest.tcl
@@ -292,7 +292,7 @@ set LOG [open releasetest-out.txt w]
 proc PUTS {args} {
   if {[llength $args]==2} {
     puts [lindex $args 0] [lindex $args 1]
-    puts [lindex $args 0] $::LOG [lindex $args 1]
+    puts $::LOG [lindex $args 1]
   } else {
     puts [lindex $args 0]
     puts $::LOG [lindex $args 0]
@@ -710,6 +710,7 @@ proc main {argv} {
   set sec [expr {$elapsetime%60}]
   set etime [format (%02d:%02d:%02d) $hr $min $sec]
   PUTS [string repeat * 79]
+  incr ::NERRCASE $::NERR
   PUTS "$::NERRCASE failures out of $::NTESTCASE tests in $etime"
   if {$::SQLITE_VERSION ne ""} {
     PUTS "SQLite $::SQLITE_VERSION"
diff --git a/test/rowid.test b/test/rowid.test
index b00b5287fd..56336453fa 100644
--- a/test/rowid.test
+++ b/test/rowid.test
@@ -144,6 +144,8 @@ do_test rowid-2.8 {
   execsql {SELECT x FROM t1 ORDER BY x}
 } {1 3 5 7 9}
 
+if 0 {  # With the index-on-expressions enhancement, creating
+        # an index on ROWID has become possible.
 # We cannot index by ROWID
 #
 do_test rowid-2.9 {
@@ -162,6 +164,7 @@ do_test rowid-2.12 {
   set v [catch {execsql {CREATE INDEX idxt1 ON t1(x, rowid)}} msg]
   lappend v $msg
 } {1 {table t1 has no column named rowid}}
+}
 
 # Columns defined in the CREATE statement override the buildin ROWID
 # column names.
diff --git a/test/subtype1.test b/test/subtype1.test
new file mode 100644
index 0000000000..1019611024
--- /dev/null
+++ b/test/subtype1.test
@@ -0,0 +1,31 @@
+# 2015-09-10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for sqlite3_value_subtype() and
+# sqlite3_result_subtype() interfaces.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test subtype1-100 {
+  SELECT test_getsubtype('hello');
+} {0}
+do_execsql_test subtype1-110 {
+  SELECT test_getsubtype(test_setsubtype('hello',123));
+} {123}
+do_execsql_test subtype1-120 {
+  SELECT typeof(test_setsubtype('hello',123));
+} {text}
+do_execsql_test subtype1-130 {
+  SELECT test_setsubtype('hello',123);
+} {hello}
+
+finish_test
diff --git a/test/tabfunc01.test b/test/tabfunc01.test
index 30a40e3138..07b3c80442 100644
--- a/test/tabfunc01.test
+++ b/test/tabfunc01.test
@@ -69,5 +69,18 @@ do_execsql_test tabfunc01-3.1 {
   SELECT DISTINCT value FROM generate_series(1,x), t1 ORDER BY 1;
 } {1 2 3}
 
+# Eponymous virtual table exists in the "main" schema only
+#
+do_execsql_test tabfunc01-4.1 {
+  SELECT * FROM main.generate_series(1,4)
+} {1 2 3 4}
+do_catchsql_test tabfunc01-4.2 {
+  SELECT * FROM temp.generate_series(1,4)
+} {1 {no such table: temp.generate_series}}
+do_catchsql_test tabfunc01-4.3 {
+  ATTACH ':memory:' AS aux1;
+  CREATE TABLE aux1.t1(a,b,c);
+  SELECT * FROM aux1.generate_series(1,4)
+} {1 {no such table: aux1.generate_series}}
 
 finish_test
diff --git a/test/table.test b/test/table.test
index 2aec6473e6..e24e3b9ed4 100644
--- a/test/table.test
+++ b/test/table.test
@@ -823,5 +823,17 @@ do_execsql_test table-18.2 {
   PRAGMA integrity_check;
 } {ok}
 
+# 2015-09-09
+# Ticket [https://www.sqlite.org/src/info/acd12990885d9276]
+# "CREATE TABLE ... AS SELECT ... FROM sqlite_master" fails because the row
+# in the sqlite_master table for the next table is initially populated
+# with a NULL instead of a record created by OP_Record.
+#
+do_execsql_test table-19.1 {
+  CREATE TABLE t19 AS SELECT * FROM sqlite_master;
+  SELECT name FROM t19 ORDER BY name;
+} {{} savepoint t10 t11 t12 t13 t16 t2 t3 t3\"xyz t4\"abc t7 t8 t9 tablet8 test1 weird}
+
+
 
 finish_test
diff --git a/test/where.test b/test/where.test
index 72fd696306..e94047564c 100644
--- a/test/where.test
+++ b/test/where.test
@@ -42,8 +42,8 @@ do_test where-1.0 {
   }
 
   execsql {
-    CREATE INDEX i1w ON t1(w);
-    CREATE INDEX i1xy ON t1(x,y);
+    CREATE INDEX i1w ON t1("w");  -- Verify quoted identifier names
+    CREATE INDEX i1xy ON t1(`x`,'y' ASC); -- Old MySQL compatibility
     CREATE INDEX i2p ON t2(p);
     CREATE INDEX i2r ON t2(r);
     CREATE INDEX i2qs ON t2(q, s);
diff --git a/test/where4.test b/test/where4.test
index 20c69771ad..3b24711514 100644
--- a/test/where4.test
+++ b/test/where4.test
@@ -136,7 +136,7 @@ do_test where4-3.1 {
     INSERT INTO t2 VALUES(1);
     INSERT INTO t2 VALUES(2);
     INSERT INTO t2 VALUES(3);
-    CREATE TABLE t3(x,y,UNIQUE(x,y));
+    CREATE TABLE t3(x,y,UNIQUE("x",'y' ASC)); -- Goofy syntax allowed
     INSERT INTO t3 VALUES(1,11);
     INSERT INTO t3 VALUES(2,NULL);
  
@@ -200,7 +200,8 @@ do_test where4-4.4 {
 ifcapable subquery {
 do_test where4-5.1 {
   execsql {
-    CREATE TABLE t4(x,y,z,PRIMARY KEY(x,y));
+    -- Allow the 'x' syntax for backwards compatibility
+    CREATE TABLE t4(x,y,z,PRIMARY KEY('x' ASC, "y" ASC));
   }
   execsql {
     SELECT *
@@ -304,4 +305,3 @@ do_execsql_test 8.2 { SELECT * FROM u9 WHERE a IS $null } {{} 1 {} 2}
 
 
 finish_test
-
diff --git a/tool/lemon.c b/tool/lemon.c
index 89d992c37e..2e8054b5cc 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -55,7 +55,7 @@ static char *msort(char*,char**,int(*)(const char*,const char*));
 ** saying they are unsafe.  So we define our own versions of those routines too.
 **
 ** There are three routines here:  lemon_sprintf(), lemon_vsprintf(), and
-** lemon_addtext().  The first two are replacements for sprintf() and vsprintf().
+** lemon_addtext(). The first two are replacements for sprintf() and vsprintf().
 ** The third is a helper routine for vsnprintf() that adds texts to the end of a
 ** buffer, making sure the buffer is always zero-terminated.
 **
@@ -316,7 +316,8 @@ enum e_action {
   RRCONFLICT,              /* Was a reduce, but part of a conflict */
   SH_RESOLVED,             /* Was a shift.  Precedence resolved conflict */
   RD_RESOLVED,             /* Was reduce.  Precedence resolved conflict */
-  NOT_USED                 /* Deleted by compression */
+  NOT_USED,                /* Deleted by compression */
+  SHIFTREDUCE              /* Shift first, then reduce */
 };
 
 /* Every shift or reduce operation is stored as one of the following */
@@ -340,7 +341,9 @@ struct state {
   struct action *ap;       /* Array of actions for this state */
   int nTknAct, nNtAct;     /* Number of actions on terminals and nonterminals */
   int iTknOfst, iNtOfst;   /* yy_action[] offset for terminals and nonterms */
-  int iDflt;               /* Default action */
+  int iDfltReduce;         /* Default action is to REDUCE by this rule */
+  struct rule *pDfltReduce;/* The default REDUCE rule. */
+  int autoReduce;          /* True if this is an auto-reduce state */
 };
 #define NO_OFFSET (-2147483647)
 
@@ -360,6 +363,7 @@ struct lemon {
   struct state **sorted;   /* Table of states sorted by state number */
   struct rule *rule;       /* List of all rules */
   int nstate;              /* Number of states */
+  int nxstate;             /* nstate with tail degenerate states removed */
   int nrule;               /* Number of rules */
   int nsymbol;             /* Number of terminal and nonterminal symbols */
   int nterminal;           /* Number of terminal symbols */
@@ -385,7 +389,8 @@ struct lemon {
   char *outname;           /* Name of the current output file */
   char *tokenprefix;       /* A prefix added to token names in the .h file */
   int nconflict;           /* Number of parsing conflicts */
-  int tablesize;           /* Size of the parse tables */
+  int nactiontab;          /* Number of entries in the yy_action[] table */
+  int tablesize;           /* Total table size of all tables in bytes */
   int basisflag;           /* Print only basis configurations */
   int has_fallback;        /* True if any %fallback is seen in the grammar */
   int nolinenosflag;       /* True if #line statements should not be printed */
@@ -483,7 +488,7 @@ static int actioncmp(
   if( rc==0 ){
     rc = (int)ap1->type - (int)ap2->type;
   }
-  if( rc==0 && ap1->type==REDUCE ){
+  if( rc==0 && (ap1->type==REDUCE || ap1->type==SHIFTREDUCE) ){
     rc = ap1->x.rp->index - ap2->x.rp->index;
   }
   if( rc==0 ){
@@ -1375,14 +1380,16 @@ void Configlist_closure(struct lemon *lemp)
 
 /* Sort the configuration list */
 void Configlist_sort(){
-  current = (struct config *)msort((char *)current,(char **)&(current->next),Configcmp);
+  current = (struct config*)msort((char*)current,(char**)&(current->next),
+                                  Configcmp);
   currentend = 0;
   return;
 }
 
 /* Sort the basis configuration list */
 void Configlist_sortbasis(){
-  basis = (struct config *)msort((char *)current,(char **)&(current->bp),Configcmp);
+  basis = (struct config*)msort((char*)current,(char**)&(current->bp),
+                                Configcmp);
   basisend = 0;
   return;
 }
@@ -1480,6 +1487,18 @@ static void handle_T_option(char *z){
   lemon_strcpy(user_templatename, z);
 }
 
+/* forward reference */
+static const char *minimum_size_type(int lwr, int upr, int *pnByte);
+
+/* Print a single line of the "Parser Stats" output
+*/
+static void stats_line(const char *zLabel, int iValue){
+  int nLabel = lemonStrlen(zLabel);
+  printf("  %s%.*s %5d\n", zLabel,
+         35-nLabel, "................................",
+         iValue);
+}
+
 /* The main program.  Parse the command line and do it... */
 int main(int argc, char **argv)
 {
@@ -1611,10 +1630,15 @@ int main(int argc, char **argv)
     if( !mhflag ) ReportHeader(&lem);
   }
   if( statistics ){
-    printf("Parser statistics: %d terminals, %d nonterminals, %d rules\n",
-      lem.nterminal, lem.nsymbol - lem.nterminal, lem.nrule);
-    printf("                   %d states, %d parser table entries, %d conflicts\n",
-      lem.nstate, lem.tablesize, lem.nconflict);
+    printf("Parser statistics:\n");
+    stats_line("terminal symbols", lem.nterminal);
+    stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal);
+    stats_line("total symbols", lem.nsymbol);
+    stats_line("rules", lem.nrule);
+    stats_line("states", lem.nxstate);
+    stats_line("conflicts", lem.nconflict);
+    stats_line("action table entries", lem.nactiontab);
+    stats_line("total table size (bytes)", lem.tablesize);
   }
   if( lem.nconflict > 0 ){
     fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict);
@@ -1873,7 +1897,8 @@ static int handleswitch(int i, FILE *err)
         dv = strtod(cp,&end);
         if( *end ){
           if( err ){
-            fprintf(err,"%sillegal character in floating-point argument.\n",emsg);
+            fprintf(err,
+               "%sillegal character in floating-point argument.\n",emsg);
             errline(i,(int)((char*)end-(char*)argv[i]),err);
           }
           errcnt++;
@@ -2939,15 +2964,14 @@ void Reprint(struct lemon *lemp)
   }
 }
 
-void ConfigPrint(FILE *fp, struct config *cfp)
-{
-  struct rule *rp;
+/* Print a single rule.
+*/
+void RulePrint(FILE *fp, struct rule *rp, int iCursor){
   struct symbol *sp;
   int i, j;
-  rp = cfp->rp;
   fprintf(fp,"%s ::=",rp->lhs->name);
   for(i=0; i<=rp->nrhs; i++){
-    if( i==cfp->dot ) fprintf(fp," *");
+    if( i==iCursor ) fprintf(fp," *");
     if( i==rp->nrhs ) break;
     sp = rp->rhs[i];
     if( sp->type==MULTITERMINAL ){
@@ -2961,6 +2985,12 @@ void ConfigPrint(FILE *fp, struct config *cfp)
   }
 }
 
+/* Print the rule for a configuration.
+*/
+void ConfigPrint(FILE *fp, struct config *cfp){
+  RulePrint(fp, cfp->rp, cfp->dot);
+}
+
 /* #define TEST */
 #if 0
 /* Print a set */
@@ -3000,15 +3030,30 @@ char *tag;
 /* Print an action to the given file descriptor.  Return FALSE if
 ** nothing was actually printed.
 */
-int PrintAction(struct action *ap, FILE *fp, int indent){
+int PrintAction(
+  struct action *ap,          /* The action to print */
+  FILE *fp,                   /* Print the action here */
+  int indent                  /* Indent by this amount */
+){
   int result = 1;
   switch( ap->type ){
-    case SHIFT:
-      fprintf(fp,"%*s shift  %d",indent,ap->sp->name,ap->x.stp->statenum);
+    case SHIFT: {
+      struct state *stp = ap->x.stp;
+      fprintf(fp,"%*s shift        %-7d",indent,ap->sp->name,stp->statenum);
       break;
-    case REDUCE:
-      fprintf(fp,"%*s reduce %d",indent,ap->sp->name,ap->x.rp->index);
+    }
+    case REDUCE: {
+      struct rule *rp = ap->x.rp;
+      fprintf(fp,"%*s reduce       %-7d",indent,ap->sp->name,rp->index);
+      RulePrint(fp, rp, -1);
       break;
+    }
+    case SHIFTREDUCE: {
+      struct rule *rp = ap->x.rp;
+      fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index);
+      RulePrint(fp, rp, -1);
+      break;
+    }
     case ACCEPT:
       fprintf(fp,"%*s accept",indent,ap->sp->name);
       break;
@@ -3017,16 +3062,16 @@ int PrintAction(struct action *ap, FILE *fp, int indent){
       break;
     case SRCONFLICT:
     case RRCONFLICT:
-      fprintf(fp,"%*s reduce %-3d ** Parsing conflict **",
+      fprintf(fp,"%*s reduce       %-7d ** Parsing conflict **",
         indent,ap->sp->name,ap->x.rp->index);
       break;
     case SSCONFLICT:
-      fprintf(fp,"%*s shift  %-3d ** Parsing conflict **", 
+      fprintf(fp,"%*s shift        %-7d ** Parsing conflict **", 
         indent,ap->sp->name,ap->x.stp->statenum);
       break;
     case SH_RESOLVED:
       if( showPrecedenceConflict ){
-        fprintf(fp,"%*s shift  %-3d -- dropped by precedence",
+        fprintf(fp,"%*s shift        %-7d -- dropped by precedence",
                 indent,ap->sp->name,ap->x.stp->statenum);
       }else{
         result = 0;
@@ -3034,7 +3079,7 @@ int PrintAction(struct action *ap, FILE *fp, int indent){
       break;
     case RD_RESOLVED:
       if( showPrecedenceConflict ){
-        fprintf(fp,"%*s reduce %-3d -- dropped by precedence",
+        fprintf(fp,"%*s reduce %-7d -- dropped by precedence",
                 indent,ap->sp->name,ap->x.rp->index);
       }else{
         result = 0;
@@ -3047,7 +3092,7 @@ int PrintAction(struct action *ap, FILE *fp, int indent){
   return result;
 }
 
-/* Generate the "y.output" log file */
+/* Generate the "*.out" log file */
 void ReportOutput(struct lemon *lemp)
 {
   int i;
@@ -3058,7 +3103,7 @@ void ReportOutput(struct lemon *lemp)
 
   fp = file_open(lemp,".out","wb");
   if( fp==0 ) return;
-  for(i=0; instate; i++){
+  for(i=0; inxstate; i++){
     stp = lemp->sorted[i];
     fprintf(fp,"State %d:\n",stp->statenum);
     if( lemp->basisflag ) cfp=stp->bp;
@@ -3166,10 +3211,11 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap)
 {
   int act;
   switch( ap->type ){
-    case SHIFT:  act = ap->x.stp->statenum;            break;
-    case REDUCE: act = ap->x.rp->index + lemp->nstate; break;
-    case ERROR:  act = lemp->nstate + lemp->nrule;     break;
-    case ACCEPT: act = lemp->nstate + lemp->nrule + 1; break;
+    case SHIFT:  act = ap->x.stp->statenum;                        break;
+    case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate;        break;
+    case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break;
+    case ERROR:  act = lemp->nstate + lemp->nrule*2;               break;
+    case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1;           break;
     default:     act = -1; break;
   }
   return act;
@@ -3228,7 +3274,8 @@ PRIVATE FILE *tplt_open(struct lemon *lemp)
     }
     in = fopen(user_templatename,"rb");
     if( in==0 ){
-      fprintf(stderr,"Can't open the template file \"%s\".\n",user_templatename);
+      fprintf(stderr,"Can't open the template file \"%s\".\n",
+              user_templatename);
       lemp->errorcnt++;
       return 0;
     }
@@ -3313,7 +3360,10 @@ void emit_destructor_code(
  }else if( sp->destructor ){
    cp = sp->destructor;
    fprintf(out,"{\n"); (*lineno)++;
-   if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,sp->destLineno,lemp->filename); }
+   if( !lemp->nolinenosflag ){
+     (*lineno)++;
+     tplt_linedir(out,sp->destLineno,lemp->filename);
+   }
  }else if( lemp->vardest ){
    cp = lemp->vardest;
    if( cp==0 ) return;
@@ -3510,13 +3560,19 @@ PRIVATE void emit_code(
 
  /* Generate code to do the reduce action */
  if( rp->code ){
-   if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,rp->line,lemp->filename); }
+   if( !lemp->nolinenosflag ){
+     (*lineno)++;
+     tplt_linedir(out,rp->line,lemp->filename);
+   }
    fprintf(out,"{%s",rp->code);
    for(cp=rp->code; *cp; cp++){
      if( *cp=='\n' ) (*lineno)++;
    } /* End loop */
    fprintf(out,"}\n"); (*lineno)++;
-   if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); }
+   if( !lemp->nolinenosflag ){
+     (*lineno)++;
+     tplt_linedir(out,*lineno,lemp->outname);
+   }
  } /* End if( rp->code ) */
 
  return;
@@ -3647,24 +3703,32 @@ void print_stack_union(
 
 /*
 ** Return the name of a C datatype able to represent values between
-** lwr and upr, inclusive.
+** lwr and upr, inclusive.  If pnByte!=NULL then also write the sizeof
+** for that type (1, 2, or 4) into *pnByte.
 */
-static const char *minimum_size_type(int lwr, int upr){
+static const char *minimum_size_type(int lwr, int upr, int *pnByte){
+  const char *zType = "int";
+  int nByte = 4;
   if( lwr>=0 ){
     if( upr<=255 ){
-      return "unsigned char";
+      zType = "unsigned char";
+      nByte = 1;
     }else if( upr<65535 ){
-      return "unsigned short int";
+      zType = "unsigned short int";
+      nByte = 2;
     }else{
-      return "unsigned int";
+      zType = "unsigned int";
+      nByte = 4;
     }
   }else if( lwr>=-127 && upr<=127 ){
-    return "signed char";
+    zType = "signed char";
+    nByte = 1;
   }else if( lwr>=-32767 && upr<32767 ){
-    return "short";
-  }else{
-    return "int";
+    zType = "short";
+    nByte = 2;
   }
+  if( pnByte ) *pnByte = nByte;
+  return zType;
 }
 
 /*
@@ -3689,7 +3753,7 @@ static int axset_compare(const void *a, const void *b){
   int c;
   c = p2->nAction - p1->nAction;
   if( c==0 ){
-    c = p2->iOrder - p1->iOrder;
+    c = p1->iOrder - p2->iOrder;
   }
   assert( c!=0 || p1==p2 );
   return c;
@@ -3728,7 +3792,9 @@ void ReportTable(
   struct action *ap;
   struct rule *rp;
   struct acttab *pActtab;
-  int i, j, n;
+  int i, j, n, sz;
+  int szActionType;     /* sizeof(YYACTIONTYPE) */
+  int szCodeType;       /* sizeof(YYCODETYPE)   */
   const char *name;
   int mnTknOfst, mxTknOfst;
   int mnNtOfst, mxNtOfst;
@@ -3769,10 +3835,10 @@ void ReportTable(
 
   /* Generate the defines */
   fprintf(out,"#define YYCODETYPE %s\n",
-    minimum_size_type(0, lemp->nsymbol+1)); lineno++;
+    minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++;
   fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1);  lineno++;
   fprintf(out,"#define YYACTIONTYPE %s\n",
-    minimum_size_type(0, lemp->nstate+lemp->nrule+5));  lineno++;
+    minimum_size_type(0,lemp->nstate+lemp->nrule*2+5,&szActionType)); lineno++;
   if( lemp->wildcard ){
     fprintf(out,"#define YYWILDCARD %d\n",
        lemp->wildcard->index); lineno++;
@@ -3808,36 +3874,24 @@ void ReportTable(
   if( mhflag ){
     fprintf(out,"#endif\n"); lineno++;
   }
-  fprintf(out,"#define YYNSTATE %d\n",lemp->nstate);  lineno++;
-  fprintf(out,"#define YYNRULE %d\n",lemp->nrule);  lineno++;
   if( lemp->errsym->useCnt ){
-    fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index);  lineno++;
-    fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum);  lineno++;
+    fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++;
+    fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++;
   }
   if( lemp->has_fallback ){
     fprintf(out,"#define YYFALLBACK 1\n");  lineno++;
   }
-  tplt_xfer(lemp->name,in,out,&lineno);
 
-  /* Generate the action table and its associates:
-  **
-  **  yy_action[]        A single table containing all actions.
-  **  yy_lookahead[]     A table containing the lookahead for each entry in
-  **                     yy_action.  Used to detect hash collisions.
-  **  yy_shift_ofst[]    For each state, the offset into yy_action for
-  **                     shifting terminals.
-  **  yy_reduce_ofst[]   For each state, the offset into yy_action for
-  **                     shifting non-terminals after a reduce.
-  **  yy_default[]       Default action for each state.
+  /* Compute the action table, but do not output it yet.  The action
+  ** table must be computed before generating the YYNSTATE macro because
+  ** we need to know how many states can be eliminated.
   */
-
-  /* Compute the actions on all states and count them up */
-  ax = (struct axset *) calloc(lemp->nstate*2, sizeof(ax[0]));
+  ax = (struct axset *) calloc(lemp->nxstate*2, sizeof(ax[0]));
   if( ax==0 ){
     fprintf(stderr,"malloc failed\n");
     exit(1);
   }
-  for(i=0; instate; i++){
+  for(i=0; inxstate; i++){
     stp = lemp->sorted[i];
     ax[i*2].stp = stp;
     ax[i*2].isTkn = 1;
@@ -3848,15 +3902,12 @@ void ReportTable(
   }
   mxTknOfst = mnTknOfst = 0;
   mxNtOfst = mnNtOfst = 0;
-
-  /* Compute the action table.  In order to try to keep the size of the
-  ** action table to a minimum, the heuristic of placing the largest action
-  ** sets first is used.
-  */
-  for(i=0; instate*2; i++) ax[i].iOrder = i;
-  qsort(ax, lemp->nstate*2, sizeof(ax[0]), axset_compare);
+  /* In an effort to minimize the action table size, use the heuristic
+  ** of placing the largest action sets first */
+  for(i=0; inxstate*2; i++) ax[i].iOrder = i;
+  qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare);
   pActtab = acttab_alloc();
-  for(i=0; instate*2 && ax[i].nAction>0; i++){
+  for(i=0; inxstate*2 && ax[i].nAction>0; i++){
     stp = ax[i].stp;
     if( ax[i].isTkn ){
       for(ap=stp->ap; ap; ap=ap->next){
@@ -3882,11 +3933,50 @@ void ReportTable(
       if( stp->iNtOfstiNtOfst;
       if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst;
     }
+#if 0  /* Uncomment for a trace of how the yy_action[] table fills out */
+    { int jj, nn;
+      for(jj=nn=0; jjnAction; jj++){
+        if( pActtab->aAction[jj].action<0 ) nn++;
+      }
+      printf("%4d: State %3d %s n: %2d size: %5d freespace: %d\n",
+             i, stp->statenum, ax[i].isTkn ? "Token" : "Var  ",
+             ax[i].nAction, pActtab->nAction, nn);
+    }
+#endif
   }
   free(ax);
 
+  /* Finish rendering the constants now that the action table has
+  ** been computed */
+  fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
+  fprintf(out,"#define YYNRULE              %d\n",lemp->nrule);  lineno++;
+  fprintf(out,"#define YY_MAX_SHIFT         %d\n",lemp->nxstate-1); lineno++;
+  fprintf(out,"#define YY_MIN_SHIFTREDUCE   %d\n",lemp->nstate); lineno++;
+  i = lemp->nstate + lemp->nrule;
+  fprintf(out,"#define YY_MAX_SHIFTREDUCE   %d\n", i-1); lineno++;
+  fprintf(out,"#define YY_MIN_REDUCE        %d\n", i); lineno++;
+  i = lemp->nstate + lemp->nrule*2;
+  fprintf(out,"#define YY_MAX_REDUCE        %d\n", i-1); lineno++;
+  fprintf(out,"#define YY_ERROR_ACTION      %d\n", i); lineno++;
+  fprintf(out,"#define YY_ACCEPT_ACTION     %d\n", i+1); lineno++;
+  fprintf(out,"#define YY_NO_ACTION         %d\n", i+2); lineno++;
+  tplt_xfer(lemp->name,in,out,&lineno);
+
+  /* Now output the action table and its associates:
+  **
+  **  yy_action[]        A single table containing all actions.
+  **  yy_lookahead[]     A table containing the lookahead for each entry in
+  **                     yy_action.  Used to detect hash collisions.
+  **  yy_shift_ofst[]    For each state, the offset into yy_action for
+  **                     shifting terminals.
+  **  yy_reduce_ofst[]   For each state, the offset into yy_action for
+  **                     shifting non-terminals after a reduce.
+  **  yy_default[]       Default action for each state.
+  */
+
   /* Output the yy_action table */
-  n = acttab_size(pActtab);
+  lemp->nactiontab = n = acttab_size(pActtab);
+  lemp->tablesize += n*szActionType;
   fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++;
   fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++;
   for(i=j=0; itablesize += n*szCodeType;
   fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++;
   for(i=j=0; instate;
+  n = lemp->nxstate;
   while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--;
   fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++;
   fprintf(out, "#define YY_SHIFT_MIN   (%d)\n", mnTknOfst); lineno++;
   fprintf(out, "#define YY_SHIFT_MAX   (%d)\n", mxTknOfst); lineno++;
   fprintf(out, "static const %s yy_shift_ofst[] = {\n", 
-          minimum_size_type(mnTknOfst-1, mxTknOfst)); lineno++;
+          minimum_size_type(mnTknOfst-1, mxTknOfst, &sz)); lineno++;
+  lemp->tablesize += n*sz;
   for(i=j=0; isorted[i];
@@ -3946,13 +4038,14 @@ void ReportTable(
 
   /* Output the yy_reduce_ofst[] table */
   fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++;
-  n = lemp->nstate;
+  n = lemp->nxstate;
   while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--;
   fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++;
   fprintf(out, "#define YY_REDUCE_MIN   (%d)\n", mnNtOfst); lineno++;
   fprintf(out, "#define YY_REDUCE_MAX   (%d)\n", mxNtOfst); lineno++;
   fprintf(out, "static const %s yy_reduce_ofst[] = {\n", 
-          minimum_size_type(mnNtOfst-1, mxNtOfst)); lineno++;
+          minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++;
+  lemp->tablesize += n*sz;
   for(i=j=0; isorted[i];
@@ -3971,11 +4064,12 @@ void ReportTable(
 
   /* Output the default action table */
   fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++;
-  n = lemp->nstate;
+  n = lemp->nxstate;
+  lemp->tablesize += n*szActionType;
   for(i=j=0; isorted[i];
     if( j==0 ) fprintf(out," /* %5d */ ", i);
-    fprintf(out, " %4d,", stp->iDflt);
+    fprintf(out, " %4d,", stp->iDfltReduce+lemp->nstate+lemp->nrule);
     if( j==9 || i==n-1 ){
       fprintf(out, "\n"); lineno++;
       j = 0;
@@ -3991,6 +4085,7 @@ void ReportTable(
   if( lemp->has_fallback ){
     int mx = lemp->nterminal - 1;
     while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; }
+    lemp->tablesize += (mx+1)*szCodeType;
     for(i=0; i<=mx; i++){
       struct symbol *p = lemp->symbols[i];
       if( p->fallback==0 ){
@@ -4255,6 +4350,32 @@ void CompressTables(struct lemon *lemp)
       if( ap->type==REDUCE && ap->x.rp==rbest ) ap->type = NOT_USED;
     }
     stp->ap = Action_sort(stp->ap);
+
+    for(ap=stp->ap; ap; ap=ap->next){
+      if( ap->type==SHIFT ) break;
+      if( ap->type==REDUCE && ap->x.rp!=rbest ) break;
+    }
+    if( ap==0 ){
+      stp->autoReduce = 1;
+      stp->pDfltReduce = rbest;
+    }
+  }
+
+  /* Make a second pass over all states and actions.  Convert
+  ** every action that is a SHIFT to an autoReduce state into
+  ** a SHIFTREDUCE action.
+  */
+  for(i=0; instate; i++){
+    stp = lemp->sorted[i];
+    for(ap=stp->ap; ap; ap=ap->next){
+      struct state *pNextState;
+      if( ap->type!=SHIFT ) continue;
+      pNextState = ap->x.stp;
+      if( pNextState->autoReduce && pNextState->pDfltReduce!=0 ){
+        ap->type = SHIFTREDUCE;
+        ap->x.rp = pNextState->pDfltReduce;
+      }
+    }
   }
 }
 
@@ -4295,17 +4416,19 @@ void ResortStates(struct lemon *lemp)
   for(i=0; instate; i++){
     stp = lemp->sorted[i];
     stp->nTknAct = stp->nNtAct = 0;
-    stp->iDflt = lemp->nstate + lemp->nrule;
+    stp->iDfltReduce = lemp->nrule;  /* Init dflt action to "syntax error" */
     stp->iTknOfst = NO_OFFSET;
     stp->iNtOfst = NO_OFFSET;
     for(ap=stp->ap; ap; ap=ap->next){
-      if( compute_action(lemp,ap)>=0 ){
+      int iAction = compute_action(lemp,ap);
+      if( iAction>=0 ){
         if( ap->sp->indexnterminal ){
           stp->nTknAct++;
         }else if( ap->sp->indexnsymbol ){
           stp->nNtAct++;
         }else{
-          stp->iDflt = compute_action(lemp, ap);
+          assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp );
+          stp->iDfltReduce = iAction - lemp->nstate - lemp->nrule;
         }
       }
     }
@@ -4315,6 +4438,10 @@ void ResortStates(struct lemon *lemp)
   for(i=0; instate; i++){
     lemp->sorted[i]->statenum = i;
   }
+  lemp->nxstate = lemp->nstate;
+  while( lemp->nxstate>1 && lemp->sorted[lemp->nxstate-1]->autoReduce ){
+    lemp->nxstate--;
+  }
 }
 
 
diff --git a/tool/lempar.c b/tool/lempar.c
index fe56d2dc16..cdf4ca5a1a 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -50,15 +50,19 @@
 **    ParseARG_PDECL     A parameter declaration for the %extra_argument
 **    ParseARG_STORE     Code to store %extra_argument into yypParser
 **    ParseARG_FETCH     Code to extract %extra_argument from yypParser
-**    YYNSTATE           the combined number of states.
-**    YYNRULE            the number of rules in the grammar
 **    YYERRORSYMBOL      is the code number of the error symbol.  If not
 **                       defined, then do no error processing.
+**    YYNSTATE           the combined number of states.
+**    YYNRULE            the number of rules in the grammar
+**    YY_MAX_SHIFT       Maximum value for shift actions
+**    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
+**    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
+**    YY_MIN_REDUCE      Maximum value for reduce actions
+**    YY_ERROR_ACTION    The yy_action[] code for syntax error
+**    YY_ACCEPT_ACTION   The yy_action[] code for accept
+**    YY_NO_ACTION       The yy_action[] code for no-op
 */
 %%
-#define YY_NO_ACTION      (YYNSTATE+YYNRULE+2)
-#define YY_ACCEPT_ACTION  (YYNSTATE+YYNRULE+1)
-#define YY_ERROR_ACTION   (YYNSTATE+YYNRULE)
 
 /* The yyzerominor constant is used to initialize instances of
 ** YYMINORTYPE objects to zero. */
@@ -85,16 +89,20 @@ static const YYMINORTYPE yyzerominor = { 0 };
 ** Suppose the action integer is N.  Then the action is determined as
 ** follows
 **
-**   0 <= N < YYNSTATE                  Shift N.  That is, push the lookahead
+**   0 <= N <= YY_MAX_SHIFT             Shift N.  That is, push the lookahead
 **                                      token onto the stack and goto state N.
 **
-**   YYNSTATE <= N < YYNSTATE+YYNRULE   Reduce by rule N-YYNSTATE.
+**   N between YY_MIN_SHIFTREDUCE       Shift to an arbitrary state then
+**     and YY_MAX_SHIFTREDUCE           reduce by rule N-YY_MIN_SHIFTREDUCE.
 **
-**   N == YYNSTATE+YYNRULE              A syntax error has occurred.
+**   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
+**     and YY_MAX_REDUCE
+
+**   N == YY_ERROR_ACTION               A syntax error has occurred.
 **
-**   N == YYNSTATE+YYNRULE+1            The parser accepts its input.
+**   N == YY_ACCEPT_ACTION              The parser accepts its input.
 **
-**   N == YYNSTATE+YYNRULE+2            No such action.  Denotes unused
+**   N == YY_NO_ACTION                  No such action.  Denotes unused
 **                                      slots in the yy_action[] table.
 **
 ** The action table is constructed as a single large table named yy_action[].
@@ -153,9 +161,13 @@ static const YYCODETYPE yyFallback[] = {
 **   +  The semantic value stored at this level of the stack.  This is
 **      the information used by the action routines in the grammar.
 **      It is sometimes called the "minor" token.
+**
+** After the "shift" half of a SHIFTREDUCE action, the stateno field
+** actually contains the reduce action for the second half of the
+** SHIFTREDUCE.
 */
 struct yyStackEntry {
-  YYACTIONTYPE stateno;  /* The state-number */
+  YYACTIONTYPE stateno;  /* The state-number, or reduce action in SHIFTREDUCE */
   YYCODETYPE major;      /* The major token value.  This is the code
                          ** number for the token at this stack level */
   YYMINORTYPE minor;     /* The user-supplied minor token value.  This
@@ -384,11 +396,11 @@ static int yy_find_shift_action(
 ){
   int i;
   int stateno = pParser->yystack[pParser->yyidx].stateno;
- 
-  if( stateno>YY_SHIFT_COUNT
-   || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){
-    return yy_default[stateno];
-  }
+
+  if( stateno>=YY_MIN_REDUCE ) return stateno; 
+  assert( stateno <= YY_SHIFT_COUNT );
+  i = yy_shift_ofst[stateno];
+  if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
   assert( iLookAhead!=YYNOCODE );
   i += iLookAhead;
   if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
@@ -489,7 +501,29 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
 }
 
 /*
-** Perform a shift action.
+** Print tracing information for a SHIFT action
+*/
+#ifndef NDEBUG
+static void yyTraceShift(yyParser *yypParser, int yyNewState){
+  if( yyTraceFILE ){
+    int i;
+    if( yyNewStateyyidx; i++)
+        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
+      fprintf(yyTraceFILE,"\n");
+    }else{
+      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
+    }
+  }
+}
+#else
+# define yyTraceShift(X,Y)
+#endif
+
+/*
+** Perform a shift action.  Return the number of errors.
 */
 static void yy_shift(
   yyParser *yypParser,          /* The parser to be shifted */
@@ -522,16 +556,7 @@ static void yy_shift(
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor = *yypMinor;
-#ifndef NDEBUG
-  if( yyTraceFILE && yypParser->yyidx>0 ){
-    int i;
-    fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
-    fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
-    for(i=1; i<=yypParser->yyidx; i++)
-      fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
-    fprintf(yyTraceFILE,"\n");
-  }
-#endif
+  yyTraceShift(yypParser, yyNewState);
 }
 
 /* The following table contains information about every rule that
@@ -564,8 +589,9 @@ static void yy_reduce(
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno>=0 
         && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
-    fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt,
-      yyRuleName[yyruleno]);
+    yysize = yyRuleInfo[yyruleno].nrhs;
+    fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt,
+      yyRuleName[yyruleno], yymsp[-yysize].stateno);
   }
 #endif /* NDEBUG */
 
@@ -602,9 +628,9 @@ static void yy_reduce(
   yysize = yyRuleInfo[yyruleno].nrhs;
   yypParser->yyidx -= yysize;
   yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
-  if( yyact < YYNSTATE ){
-#ifdef NDEBUG
-    /* If we are not debugging and the reduce action popped at least
+  if( yyact <= YY_MAX_SHIFTREDUCE ){
+    if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+    /* If the reduce action popped at least
     ** one element off the stack, then we can push the new element back
     ** onto the stack here, and skip the stack overflow test in yy_shift().
     ** That gives a significant speed improvement. */
@@ -614,13 +640,12 @@ static void yy_reduce(
       yymsp->stateno = (YYACTIONTYPE)yyact;
       yymsp->major = (YYCODETYPE)yygoto;
       yymsp->minor = yygotominor;
-    }else
-#endif
-    {
+      yyTraceShift(yypParser, yyact);
+    }else{
       yy_shift(yypParser,yyact,yygoto,&yygotominor);
     }
   }else{
-    assert( yyact == YYNSTATE + YYNRULE + 1 );
+    assert( yyact == YY_ACCEPT_ACTION );
     yy_accept(yypParser);
   }
 }
@@ -740,13 +765,13 @@ void Parse(
 
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
-    if( yyact YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
       yy_shift(yypParser,yyact,yymajor,&yyminorunion);
       yypParser->yyerrcnt--;
       yymajor = YYNOCODE;
-    }else if( yyact < YYNSTATE + YYNRULE ){
-      yy_reduce(yypParser,yyact-YYNSTATE);
+    }else if( yyact <= YY_MAX_REDUCE ){
+      yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
     }else{
       assert( yyact == YY_ERROR_ACTION );
 #ifdef YYERRORSYMBOL
@@ -796,7 +821,7 @@ void Parse(
           yymx != YYERRORSYMBOL &&
           (yyact = yy_find_reduce_action(
                         yypParser->yystack[yypParser->yyidx].stateno,
-                        YYERRORSYMBOL)) >= YYNSTATE
+                        YYERRORSYMBOL)) >= YY_MIN_REDUCE
         ){
           yy_pop_parser_stack(yypParser);
         }
@@ -846,5 +871,10 @@ void Parse(
 #endif
     }
   }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
+#ifndef NDEBUG
+  if( yyTraceFILE ){
+    fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt);
+  }
+#endif
   return;
 }
diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl
index dea2723397..38d954162e 100644
--- a/tool/spaceanal.tcl
+++ b/tool/spaceanal.tcl
@@ -26,7 +26,19 @@ proc is_without_rowid {tname} {
 #
 proc usage {} {
   set argv0 [file rootname [file tail [info nameofexecutable]]]
-  puts stderr "Usage: $argv0 \[--pageinfo] \[--stats] database-name"
+  puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename"
+  puts stderr {
+Analyze the SQLite3 database file specified by the "database-filename"
+argument and output a report detailing size and storage efficiency
+information for the database and its constituent tables and indexes.
+
+Options:
+
+   --stats        Output SQL text that creates a new database containing
+                  statistics about the database that was analyzed
+
+   --pageinfo     Show how each page of the database-file is used
+}
   exit 1
 }
 set file_to_analyze {}