mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge the fts4-content branch with the trunk.

Browse Source 
FossilOrigin-Name: 8a4077057ddeb08e8edc5f20a75abaaba7a278ba
This commit is contained in:
dan 2011-10-19 16:20:40 +00:00
commit e459b07601
17 changed files with 1276 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

421
ext/fts3/fts3.c
View File

@ -468,6 +468,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
sqlite3_free(p->zSegmentsTbl);
sqlite3_free(p->zReadExprlist);
sqlite3_free(p->zWriteExprlist);
sqlite3_free(p->zContentTbl);
/* Invoke the tokenizer destructor to free the tokenizer. */
p->pTokenizer->pModule->xDestroy(p->pTokenizer);
@ -507,16 +508,19 @@ static void fts3DbExec(
** The xDestroy() virtual table method.
*/
static int fts3DestroyMethod(sqlite3_vtab *pVtab){
int rc = SQLITE_OK; /* Return code */
Fts3Table *p = (Fts3Table *)pVtab;
sqlite3 *db = p->db;
int rc = SQLITE_OK; /* Return code */
const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */
sqlite3 *db = p->db; /* Database handle */
/* Drop the shadow tables */
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName);
if( p->zContentTbl==0 ){
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName);
}
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName);
fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName);
/* If everything has worked, invoke fts3DisconnectMethod() to free the
** memory associated with the Fts3Table structure and return SQLITE_OK.
@ -578,23 +582,27 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
static int fts3CreateTables(Fts3Table *p){
int rc = SQLITE_OK; /* Return code */
int i; /* Iterator variable */
char *zContentCols; /* Columns of %_content table */
sqlite3 *db = p->db; /* The database connection */
/* Create a list of user columns for the content table */
zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
for(i=0; zContentCols && i<p->nColumn; i++){
char *z = p->azColumn[i];
zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
if( zContentCols==0 ) rc = SQLITE_NOMEM;
if( p->zContentTbl==0 ){
char *zContentCols; /* Columns of %_content table */
/* Create a list of user columns for the content table */
zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
for(i=0; zContentCols && i<p->nColumn; i++){
char *z = p->azColumn[i];
zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
if( zContentCols==0 ) rc = SQLITE_NOMEM;
/* Create the content table */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_content'(%s)",
p->zDb, p->zName, zContentCols
);
sqlite3_free(zContentCols);
}
/* Create the content table */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_content'(%s)",
p->zDb, p->zName, zContentCols
);
sqlite3_free(zContentCols);
/* Create other tables */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
@ -745,8 +753,8 @@ static char *fts3QuoteId(char const *zInput){
}
/*
** Return a list of comma separated SQL expressions that could be used
** in a SELECT statement such as the following:
** Return a list of comma separated SQL expressions and a FROM clause that
** could be used in a SELECT statement such as the following:
**
** SELECT <list of expressions> FROM %_content AS x ...
**
@ -757,7 +765,7 @@ static char *fts3QuoteId(char const *zInput){
** table has the three user-defined columns "a", "b", and "c", the following
** string is returned:
**
** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')"
** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x"
**
** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
** is the responsibility of the caller to eventually free it.
@ -773,16 +781,28 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
char *zFunction;
int i;
if( !zFunc ){
zFunction = "";
if( p->zContentTbl==0 ){
if( !zFunc ){
zFunction = "";
}else{
zFree = zFunction = fts3QuoteId(zFunc);
}
fts3Appendf(pRc, &zRet, "docid");
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
}
sqlite3_free(zFree);
}else{
zFree = zFunction = fts3QuoteId(zFunc);
fts3Appendf(pRc, &zRet, "rowid");
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
}
}
fts3Appendf(pRc, &zRet, "docid");
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
}
sqlite3_free(zFree);
fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x",
p->zDb,
(p->zContentTbl ? p->zContentTbl : p->zName),
(p->zContentTbl ? "" : "_content")
);
return zRet;
}
@ -906,6 +926,91 @@ static int fts3PrefixParameter(
return SQLITE_OK;
}
/*
** This function is called when initializing an FTS4 table that uses the
** content=xxx option. It determines the number of and names of the columns
** of the new FTS4 table.
**
** The third argument passed to this function is the value passed to the
** config=xxx option (i.e. "xxx"). This function queries the database for
** a table of that name. If found, the output variables are populated
** as follows:
**
** *pnCol: Set to the number of columns table xxx has,
**
** *pnStr: Set to the total amount of space required to store a copy
** of each columns name, including the nul-terminator.
**
** *pazCol: Set to point to an array of *pnCol strings. Each string is
** the name of the corresponding column in table xxx. The array
** and its contents are allocated using a single allocation. It
** is the responsibility of the caller to free this allocation
** by eventually passing the *pazCol value to sqlite3_free().
**
** If the table cannot be found, an error code is returned and the output
** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is
** returned (and the output variables are undefined).
*/
static int fts3ContentColumns(
sqlite3 *db, /* Database handle */
const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */
const char *zTbl, /* Name of content table */
const char ***pazCol, /* OUT: Malloc'd array of column names */
int *pnCol, /* OUT: Size of array *pazCol */
int *pnStr /* OUT: Bytes of string content */
){
int rc = SQLITE_OK; /* Return code */
char *zSql; /* "SELECT *" statement on zTbl */
sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */
zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
}
sqlite3_free(zSql);
if( rc==SQLITE_OK ){
const char **azCol; /* Output array */
int nStr = 0; /* Size of all column names (incl. 0x00) */
int nCol; /* Number of table columns */
int i; /* Used to iterate through columns */
/* Loop through the returned columns. Set nStr to the number of bytes of
** space required to store a copy of each column name, including the
** nul-terminator byte. */
nCol = sqlite3_column_count(pStmt);
for(i=0; i<nCol; i++){
const char *zCol = sqlite3_column_name(pStmt, i);
nStr += strlen(zCol) + 1;
}
/* Allocate and populate the array to return. */
azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr);
if( azCol==0 ){
rc = SQLITE_NOMEM;
}else{
char *p = (char *)&azCol[nCol];
for(i=0; i<nCol; i++){
const char *zCol = sqlite3_column_name(pStmt, i);
int n = strlen(zCol)+1;
memcpy(p, zCol, n);
azCol[i] = p;
p += n;
}
}
sqlite3_finalize(pStmt);
/* Set the output variables. */
*pnCol = nCol;
*pnStr = nStr;
*pazCol = azCol;
}
return rc;
}
/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
@ -950,6 +1055,7 @@ static int fts3InitVtab(
char *zPrefix = 0; /* Prefix parameter value (or NULL) */
char *zCompress = 0; /* compress=? parameter (or NULL) */
char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
char *zContent = 0; /* content=? parameter (or NULL) */
assert( strlen(argv[0])==4 );
assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
@ -993,13 +1099,13 @@ static int fts3InitVtab(
struct Fts4Option {
const char *zOpt;
int nOpt;
char **pzVar;
} aFts4Opt[] = {
{ "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */
{ "prefix", 6, 0 }, /* 1 -> PREFIX */
{ "compress", 8, 0 }, /* 2 -> COMPRESS */
{ "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */
{ "order", 5, 0 } /* 4 -> ORDER */
{ "matchinfo", 9 }, /* 0 -> MATCHINFO */
{ "prefix", 6 }, /* 1 -> PREFIX */
{ "compress", 8 }, /* 2 -> COMPRESS */
{ "uncompress", 10 }, /* 3 -> UNCOMPRESS */
{ "order", 5 }, /* 4 -> ORDER */
{ "content", 7 } /* 5 -> CONTENT */
};
int iOpt;
@ -1045,13 +1151,20 @@ static int fts3InitVtab(
case 4: /* ORDER */
if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
&& (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3))
&& (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4))
){
*pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
rc = SQLITE_ERROR;
}
bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
break;
default: /* CONTENT */
assert( iOpt==5 );
sqlite3_free(zUncompress);
zContent = zVal;
zVal = 0;
break;
}
}
sqlite3_free(zVal);
@ -1064,6 +1177,26 @@ static int fts3InitVtab(
aCol[nCol++] = z;
}
}
/* If a content=xxx option was specified, the following:
**
** 1. Ignore any compress= and uncompress= options.
**
** 2. If no column names were specified as part of the CREATE VIRTUAL
** TABLE statement, use all columns from the content table.
*/
if( rc==SQLITE_OK && zContent ){
sqlite3_free(zCompress);
sqlite3_free(zUncompress);
zCompress = 0;
zUncompress = 0;
if( nCol==0 ){
sqlite3_free(aCol);
aCol = 0;
rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
}
assert( rc!=SQLITE_OK || nCol>0 );
}
if( rc!=SQLITE_OK ) goto fts3_init_out;
if( nCol==0 ){
@ -1108,6 +1241,8 @@ static int fts3InitVtab(
p->bHasDocsize = (isFts4 && bNoDocsize==0);
p->bHasStat = isFts4;
p->bDescIdx = bDescIdx;
p->zContentTbl = zContent;
zContent = 0;
TESTONLY( p->inTransaction = -1 );
TESTONLY( p->mxSavepoint = -1 );
@ -1169,6 +1304,7 @@ fts3_init_out:
sqlite3_free(aIndex);
sqlite3_free(zCompress);
sqlite3_free(zUncompress);
sqlite3_free(zContent);
sqlite3_free((void *)aCol);
if( rc!=SQLITE_OK ){
if( p ){
@ -1320,35 +1456,64 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
return SQLITE_OK;
}
/*
** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
** compose and prepare an SQL statement of the form:
**
** "SELECT <columns> FROM %_content WHERE rowid = ?"
**
** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
** it. If an error occurs, return an SQLite error code.
**
** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
*/
static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
int rc = SQLITE_OK;
if( pCsr->pStmt==0 ){
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
char *zSql;
zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
if( !zSql ) return SQLITE_NOMEM;
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
}
*ppStmt = pCsr->pStmt;
return rc;
}
/*
** Position the pCsr->pStmt statement so that it is on the row
** of the %_content table that contains the last match. Return
** SQLITE_OK on success.
*/
static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
int rc = SQLITE_OK;
if( pCsr->isRequireSeek ){
sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
pCsr->isRequireSeek = 0;
if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
return SQLITE_OK;
}else{
int rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK ){
/* If no row was found and no error has occured, then the %_content
** table is missing a row that is present in the full-text index.
** The data structures are corrupt.
*/
rc = FTS_CORRUPT_VTAB;
sqlite3_stmt *pStmt = 0;
rc = fts3CursorSeekStmt(pCsr, &pStmt);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
pCsr->isRequireSeek = 0;
if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
return SQLITE_OK;
}else{
rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){
/* If no row was found and no error has occured, then the %_content
** table is missing a row that is present in the full-text index.
** The data structures are corrupt. */
rc = FTS_CORRUPT_VTAB;
pCsr->isEof = 1;
}
}
pCsr->isEof = 1;
if( pContext ){
sqlite3_result_error_code(pContext, rc);
}
return rc;
}
}else{
return SQLITE_OK;
}
if( rc!=SQLITE_OK && pContext ){
sqlite3_result_error_code(pContext, rc);
}
return rc;
}
/*
@ -1788,7 +1953,7 @@ static int fts3PoslistPhraseMerge(
char **pp1, /* IN/OUT: Left input list */
char **pp2 /* IN/OUT: Right input list */
){
char *p = (pp ? *pp : 0);
char *p = *pp;
char *p1 = *pp1;
char *p2 = *pp2;
int iCol1 = 0;
@ -1814,7 +1979,7 @@ static int fts3PoslistPhraseMerge(
sqlite3_int64 iPos1 = 0;
sqlite3_int64 iPos2 = 0;
if( pp && iCol1 ){
if( iCol1 ){
*p++ = POS_COLUMN;
p += sqlite3Fts3PutVarint(p, iCol1);
}
@ -1829,13 +1994,6 @@ static int fts3PoslistPhraseMerge(
|| (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken)
){
sqlite3_int64 iSave;
if( !pp ){
fts3PoslistCopy(0, &p2);
fts3PoslistCopy(0, &p1);
*pp1 = p1;
*pp2 = p2;
return 1;
}
iSave = isSaveLeft ? iPos1 : iPos2;
fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
pSave = 0;
@ -1887,7 +2045,7 @@ static int fts3PoslistPhraseMerge(
fts3PoslistCopy(0, &p1);
*pp1 = p1;
*pp2 = p2;
if( !pp || *pp==p ){
if( *pp==p ){
return 0;
}
*p++ = 0x00;
@ -2190,6 +2348,56 @@ static void fts3DoclistPhraseMerge(
*pnRight = p - aOut;
}
/*
** Argument pList points to a position list nList bytes in size. This
** function checks to see if the position list contains any entries for
** a token in position 0 (of any column). If so, it writes argument iDelta
** to the output buffer pOut, followed by a position list consisting only
** of the entries from pList at position 0, and terminated by an 0x00 byte.
** The value returned is the number of bytes written to pOut (if any).
*/
int sqlite3Fts3FirstFilter(
sqlite3_int64 iDelta, /* Varint that may be written to pOut */
char *pList, /* Position list (no 0x00 term) */
int nList, /* Size of pList in bytes */
char *pOut /* Write output here */
){
int nOut = 0;
int bWritten = 0; /* True once iDelta has been written */
char *p = pList;
char *pEnd = &pList[nList];
if( *p!=0x01 ){
if( *p==0x02 ){
nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
pOut[nOut++] = 0x02;
bWritten = 1;
}
fts3ColumnlistCopy(0, &p);
}
while( p<pEnd && *p==0x01 ){
sqlite3_int64 iCol;
p++;
p += sqlite3Fts3GetVarint(p, &iCol);
if( *p==0x02 ){
if( bWritten==0 ){
nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
bWritten = 1;
}
pOut[nOut++] = 0x01;
nOut += sqlite3Fts3PutVarint(&pOut[nOut], iCol);
pOut[nOut++] = 0x02;
}
fts3ColumnlistCopy(0, &p);
}
if( bWritten ){
pOut[nOut++] = 0x00;
}
return nOut;
}
/*
** Merge all doclists in the TermSelect.aaOutput[] array into a single
@ -2546,6 +2754,7 @@ static int fts3TermSelect(
filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS
| (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
| (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0)
| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
filter.iCol = iColumn;
filter.zTerm = pTok->z;
@ -2686,8 +2895,8 @@ static int fts3FilterMethod(
return SQLITE_NOMEM;
}
rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
iCol, zQuery, -1, &pCsr->pExpr
rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat,
p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
);
if( rc!=SQLITE_OK ){
if( rc==SQLITE_ERROR ){
@ -2714,23 +2923,24 @@ static int fts3FilterMethod(
** row by docid.
*/
if( idxNum==FTS3_FULLSCAN_SEARCH ){
const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
}else{
const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
zSql = sqlite3_mprintf(
"SELECT %s ORDER BY rowid %s",
p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
);
if( zSql ){
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
}else{
rc = SQLITE_NOMEM;
}
}else if( idxNum==FTS3_DOCID_SEARCH ){
rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
if( rc==SQLITE_OK ){
rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
}
}
if( !zSql ) return SQLITE_NOMEM;
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
sqlite3_free(zSql);
if( rc!=SQLITE_OK ) return rc;
if( idxNum==FTS3_DOCID_SEARCH ){
rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
if( rc!=SQLITE_OK ) return rc;
}
return fts3NextMethod(pCursor);
}
@ -2782,7 +2992,7 @@ static int fts3ColumnMethod(
sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
}else{
rc = fts3CursorSeek(0, pCsr);
if( rc==SQLITE_OK ){
if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
}
}
@ -3075,15 +3285,22 @@ static int fts3RenameMethod(
sqlite3 *db = p->db; /* Database connection */
int rc; /* Return Code */
/* As it happens, the pending terms table is always empty here. This is
** because an "ALTER TABLE RENAME TABLE" statement inside a transaction
** always opens a savepoint transaction. And the xSavepoint() method
** flushes the pending terms table. But leave the (no-op) call to
** PendingTermsFlush() in in case that changes.
*/
assert( p->nPendingData==0 );
rc = sqlite3Fts3PendingTermsFlush(p);
if( rc!=SQLITE_OK ){
return rc;
if( p->zContentTbl==0 ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
p->zDb, p->zName, zName
);
}
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
p->zDb, p->zName, zName
);
if( p->bHasDocsize ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';",
@ -3442,21 +3659,20 @@ static int fts3EvalPhraseLoad(
*/
static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
int iToken; /* Used to iterate through phrase tokens */
int rc = SQLITE_OK; /* Return code */
char *aPoslist = 0; /* Position list for deferred tokens */
int nPoslist = 0; /* Number of bytes in aPoslist */
int iPrev = -1; /* Token number of previous deferred token */
assert( pPhrase->doclist.bFreeList==0 );
for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
for(iToken=0; iToken<pPhrase->nToken; iToken++){
Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
Fts3DeferredToken *pDeferred = pToken->pDeferred;
if( pDeferred ){
char *pList;
int nList;
rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
if( rc!=SQLITE_OK ) return rc;
if( pList==0 ){
@ -3557,6 +3773,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
&& p->nToken==1
&& pFirst->pSegcsr
&& pFirst->pSegcsr->bLookup
&& pFirst->bFirst==0
){
/* Use the incremental approach. */
int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
@ -3786,7 +4003,7 @@ static void fts3EvalTokenCosts(
Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */
int *pRc /* IN/OUT: Error code */
){
if( *pRc==SQLITE_OK && pExpr ){
if( *pRc==SQLITE_OK ){
if( pExpr->eType==FTSQUERY_PHRASE ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int i;
@ -3800,6 +4017,11 @@ static void fts3EvalTokenCosts(
*pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
}
}else if( pExpr->eType!=FTSQUERY_NOT ){
assert( pExpr->eType==FTSQUERY_OR
|| pExpr->eType==FTSQUERY_AND
|| pExpr->eType==FTSQUERY_NEAR
);
assert( pExpr->pLeft && pExpr->pRight );
if( pExpr->eType==FTSQUERY_OR ){
pRoot = pExpr->pLeft;
**ppOr = pRoot;
@ -3904,6 +4126,15 @@ static int fts3EvalSelectDeferred(
int nMinEst = 0; /* The minimum count for any phrase so far. */
int nLoad4 = 1; /* (Phrases that will be loaded)^4. */
/* Tokens are never deferred for FTS tables created using the content=xxx
** option. The reason being that it is not guaranteed that the content
** table actually contains the same data as the index. To prevent this from
** causing any problems, the deferred token optimization is completely
** disabled for content=xxx tables. */
if( pTab->zContentTbl ){
return SQLITE_OK;
}
/* Count the tokens in this AND/NEAR cluster. If none of the doclists
** associated with the tokens spill onto overflow pages, or if there is
** only 1 token, exit early. No tokens to defer in this case. */

9
ext/fts3/fts3Int.h
View File

@ -191,6 +191,7 @@ struct Fts3Table {
int nColumn; /* number of named columns in virtual table */
char **azColumn; /* column names. malloced */
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
char *zContentTbl; /* content=xxx option, or NULL */
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
@ -231,7 +232,7 @@ struct Fts3Table {
int nPendingData; /* Current bytes of pending data */
sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
#if defined(SQLITE_DEBUG)
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
/* State variables used for validating that the transaction control
** methods of the virtual table are called at appropriate times. These
** values do not contribution to the FTS computation; they are used for
@ -316,6 +317,7 @@ struct Fts3PhraseToken {
char *z; /* Text of the token */
int n; /* Number of bytes in buffer z */
int isPrefix; /* True if token ends with a "*" character */
int bFirst; /* True if token must appear at position 0 */
/* Variables above this point are populated when the expression is
** parsed (by code in fts3_expr.c). Below this point the variables are
@ -434,6 +436,7 @@ int sqlite3Fts3SegReaderCursor(
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
#define FTS3_SEGMENT_PREFIX 0x00000008
#define FTS3_SEGMENT_SCAN 0x00000010
#define FTS3_SEGMENT_FIRST 0x00000020
/* Type passed as 4th argument to SegmentReaderIterate() */
struct Fts3SegFilter {
@ -473,8 +476,8 @@ int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
@ -493,7 +496,7 @@ void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
char **, int, int, const char *, int, Fts3Expr **
char **, int, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST

22
ext/fts3/fts3_expr.c
View File

@ -93,6 +93,7 @@ typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
const char **azCol; /* Array of column names for fts3 table */
int bFts4; /* True to allow FTS4-only syntax */
int nCol; /* Number of entries in azCol[] */
int iDefaultCol; /* Default column to query */
int isNot; /* True if getNextNode() sees a unary - */
@ -180,9 +181,21 @@ static int getNextToken(
pRet->pPhrase->aToken[0].isPrefix = 1;
iEnd++;
}
if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
pParse->isNot = 1;
while( 1 ){
if( !sqlite3_fts3_enable_parentheses
&& iStart>0 && z[iStart-1]=='-'
){
pParse->isNot = 1;
iStart--;
}else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){
pRet->pPhrase->aToken[0].bFirst = 1;
iStart--;
}else{
break;
}
}
}
nConsumed = iEnd;
}
@ -281,6 +294,7 @@ static int getNextString(
pToken->n = nByte;
pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
pToken->bFirst = (iBegin>0 && zInput[iBegin-1]=='^');
nToken = ii+1;
}
}
@ -732,6 +746,7 @@ exprparse_out:
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
char **azCol, /* Array of column names for fts3 table */
int bFts4, /* True to allow FTS4-only syntax */
int nCol, /* Number of entries in azCol[] */
int iDefaultCol, /* Default column to query */
const char *z, int n, /* Text of MATCH query */
@ -745,6 +760,7 @@ int sqlite3Fts3ExprParse(
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.nNest = 0;
sParse.bFts4 = bFts4;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
@ -934,7 +950,7 @@ static void fts3ExprTest(
}
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);

5
ext/fts3/fts3_snippet.c
View File

@ -368,6 +368,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
int iFirst = 0;
pPhrase->pList = pCsr;
fts3GetDeltaPosition(&pCsr, &iFirst);
assert( iFirst>=0 );
pPhrase->pHead = pCsr;
pPhrase->pTail = pCsr;
pPhrase->iHead = iFirst;
@ -1409,7 +1410,7 @@ void sqlite3Fts3Offsets(
if( !pTerm ){
/* All offsets for this column have been gathered. */
break;
rc = SQLITE_DONE;
}else{
assert( iCurrent<=iMinPos );
if( 0==(0xFE&*pTerm->pList) ){
@ -1426,7 +1427,7 @@ void sqlite3Fts3Offsets(
"%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
);
rc = fts3StringAppend(&res, aBuffer, -1);
}else if( rc==SQLITE_DONE ){
}else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){
rc = FTS_CORRUPT_VTAB;
}
}

188
ext/fts3/fts3_write.c
View File

@ -256,7 +256,7 @@ static int fts3SqlStmt(
/* 4 */ "DELETE FROM %Q.'%q_segdir'",
/* 5 */ "DELETE FROM %Q.'%q_docsize'",
/* 6 */ "DELETE FROM %Q.'%q_stat'",
/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?",
/* 7 */ "SELECT %s WHERE rowid=?",
/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
@ -298,7 +298,7 @@ static int fts3SqlStmt(
if( eStmt==SQL_CONTENT_INSERT ){
zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
}else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName);
zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
}else{
zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
}
@ -409,17 +409,24 @@ static void fts3SqlExec(
** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can
** still happen if the user reads data directly from the %_segments or
** %_segdir tables instead of going through FTS3 though.
**
** This reasoning does not apply to a content=xxx table.
*/
int sqlite3Fts3ReadLock(Fts3Table *p){
int rc; /* Return code */
sqlite3_stmt *pStmt; /* Statement used to obtain lock */
rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_null(pStmt, 1);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
if( p->zContentTbl==0 ){
rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_null(pStmt, 1);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
}
}else{
rc = SQLITE_OK;
}
return rc;
}
@ -780,6 +787,18 @@ static int fts3InsertData(
int rc; /* Return code */
sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */
if( p->zContentTbl ){
sqlite3_value *pRowid = apVal[p->nColumn+3];
if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
pRowid = apVal[1];
}
if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
return SQLITE_CONSTRAINT;
}
*piDocid = sqlite3_value_int64(pRowid);
return SQLITE_OK;
}
/* Locate the statement handle used to insert data into the %_content
** table. The SQL for this statement is:
**
@ -830,14 +849,16 @@ static int fts3InsertData(
** Remove all data from the FTS3 table. Clear the hash table containing
** pending terms.
*/
static int fts3DeleteAll(Fts3Table *p){
static int fts3DeleteAll(Fts3Table *p, int bContent){
int rc = SQLITE_OK; /* Return code */
/* Discard the contents of the pending-terms hash table. */
sqlite3Fts3PendingTermsClear(p);
/* Delete everything from the %_content, %_segments and %_segdir tables. */
fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
/* Delete everything from the shadow tables. Except, leave %_content as
** is if bContent is false. */
assert( p->zContentTbl==0 || bContent==0 );
if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
if( p->bHasDocsize ){
@ -2125,12 +2146,18 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){
static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
sqlite3_stmt *pStmt;
int rc;
rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
if( rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pStmt) ){
*pisEmpty = sqlite3_column_int(pStmt, 0);
if( p->zContentTbl ){
/* If using the content=xxx option, assume the table is never empty */
*pisEmpty = 0;
rc = SQLITE_OK;
}else{
rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
if( rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pStmt) ){
*pisEmpty = sqlite3_column_int(pStmt, 0);
}
rc = sqlite3_reset(pStmt);
}
rc = sqlite3_reset(pStmt);
}
return rc;
}
@ -2482,6 +2509,7 @@ int sqlite3Fts3SegReaderStep(
int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST);
Fts3SegReader **apSegment = pCsr->apSegment;
int nSegment = pCsr->nSegment;
@ -2541,6 +2569,7 @@ int sqlite3Fts3SegReaderStep(
assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
if( nMerge==1
&& !isIgnoreEmpty
&& !isFirst
&& (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
){
pCsr->nDoclist = apSegment[0]->nDoclist;
@ -2606,12 +2635,24 @@ int sqlite3Fts3SegReaderStep(
}
pCsr->aBuffer = aNew;
}
nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
iPrev = iDocid;
if( isRequirePos ){
memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
nDoclist += nList;
pCsr->aBuffer[nDoclist++] = '\0';
if( isFirst ){
char *a = &pCsr->aBuffer[nDoclist];
int nWrite;
nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a);
if( nWrite ){
iPrev = iDocid;
nDoclist += nWrite;
}
}else{
nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
iPrev = iDocid;
if( isRequirePos ){
memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
nDoclist += nList;
pCsr->aBuffer[nDoclist++] = '\0';
}
}
}
@ -2787,9 +2828,9 @@ static void fts3DecodeIntArray(
** a blob of varints.
*/
static void fts3InsertDocsize(
int *pRC, /* Result code */
Fts3Table *p, /* Table into which to insert */
u32 *aSz /* Sizes of each column */
int *pRC, /* Result code */
Fts3Table *p, /* Table into which to insert */
u32 *aSz /* Sizes of each column, in tokens */
){
char *pBlob; /* The BLOB encoding of the document size */
int nBlob; /* Number of bytes in the BLOB */
@ -2911,6 +2952,86 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
}
/*
** This function is called when the user executes the following statement:
**
** INSERT INTO <tbl>(<tbl>) VALUES('rebuild');
**
** The entire FTS index is discarded and rebuilt. If the table is one
** created using the content=xxx option, then the new index is based on
** the current contents of the xxx table. Otherwise, it is rebuilt based
** on the contents of the %_content table.
*/
static int fts3DoRebuild(Fts3Table *p){
int rc; /* Return Code */
rc = fts3DeleteAll(p, 0);
if( rc==SQLITE_OK ){
u32 *aSz = 0;
u32 *aSzIns;
u32 *aSzDel;
sqlite3_stmt *pStmt = 0;
int nEntry = 0;
/* Compose and prepare an SQL statement to loop through the content table */
char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
sqlite3_free(zSql);
}
if( rc==SQLITE_OK ){
int nByte = sizeof(u32) * (p->nColumn+1)*3;
aSz = (u32 *)sqlite3_malloc(nByte);
if( aSz==0 ){
rc = SQLITE_NOMEM;
}else{
memset(aSz, 0, nByte);
aSzIns = &aSz[p->nColumn+1];
aSzDel = &aSzIns[p->nColumn+1];
}
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int iCol;
rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0));
aSz[p->nColumn] = 0;
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]);
aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
}
if( p->bHasDocsize ){
fts3InsertDocsize(&rc, p, aSz);
}
if( rc!=SQLITE_OK ){
sqlite3_finalize(pStmt);
pStmt = 0;
}else{
nEntry++;
for(iCol=0; iCol<=p->nColumn; iCol++){
aSzIns[iCol] += aSz[iCol];
}
}
}
if( p->bHasStat ){
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry);
}
sqlite3_free(aSz);
if( pStmt ){
int rc2 = sqlite3_finalize(pStmt);
if( rc==SQLITE_OK ){
rc = rc2;
}
}
}
return rc;
}
/*
** Handle a 'special' INSERT of the form:
**
@ -2928,6 +3049,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
return SQLITE_NOMEM;
}else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
rc = fts3DoOptimize(p, 0);
}else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
rc = fts3DoRebuild(p);
#ifdef SQLITE_TEST
}else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
p->nNodeSize = atoi(&zVal[9]);
@ -3008,6 +3131,7 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
Fts3PhraseToken *pPT = pDef->pToken;
if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
&& (pPT->bFirst==0 || iPos==0)
&& (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
&& (0==memcmp(zToken, pPT->z, pPT->n))
){
@ -3099,14 +3223,18 @@ static int fts3DeleteByRowid(
/* Deleting this row means the whole table is empty. In this case
** delete the contents of all three tables and throw away any
** data in the pendingTerms hash table. */
rc = fts3DeleteAll(p);
rc = fts3DeleteAll(p, 1);
*pnDoc = *pnDoc - 1;
}else{
sqlite3_int64 iRemove = sqlite3_value_int64(pRowid);
rc = fts3PendingTermsDocid(p, iRemove);
fts3DeleteTerms(&rc, p, pRowid, aSzDel);
fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1;
if( p->zContentTbl==0 ){
fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1;
}else{
*pnDoc = *pnDoc - 1;
}
if( p->bHasDocsize ){
fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
}
@ -3166,7 +3294,7 @@ int sqlite3Fts3UpdateMethod(
** detect the conflict and return SQLITE_CONSTRAINT before beginning to
** modify the database file.
*/
if( nArg>1 ){
if( nArg>1 && p->zContentTbl==0 ){
/* Find the value object that holds the new rowid value. */
sqlite3_value *pNewRowid = apVal[3+p->nColumn];
if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
@ -3217,7 +3345,9 @@ int sqlite3Fts3UpdateMethod(
if( nArg>1 && rc==SQLITE_OK ){
if( bInsertDone==0 ){
rc = fts3InsertData(p, apVal, pRowid);
if( rc==SQLITE_CONSTRAINT ) rc = FTS_CORRUPT_VTAB;
if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
rc = FTS_CORRUPT_VTAB;
}
}
if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
rc = fts3PendingTermsDocid(p, *pRowid);

40
manifest
View File

@ -1,5 +1,5 @@
C Fix\sthe\svirtual\stable\srename\slogic\sso\sthat\sit\sworks\seven\sif\sthe\sdatabase\nencoding\sis\ssomething\sother\sthan\sUTF8.\nTicket\s[8290242b2a9a81683]
D 2011-10-18T22:07:47.722
C Merge\sthe\sfts4-content\sbranch\swith\sthe\strunk.
D 2011-10-19T16:20:40.613
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -62,22 +62,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 19f36945148cfd4ee3655b5fac0879ba4e0f3117
F ext/fts3/fts3.c 9c125699baf1c03a3d8d53fb7d8d27ab4ad7d6eb
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 59c5a9475fed5d76c70a4763103b3c8e60424a68
F ext/fts3/fts3Int.h def7a900f98c5ab5fa4772e922bfa219d5097f05
F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691
F ext/fts3/fts3_expr.c 61ceee7c9698c7ae1ba55ab6ff7961456a8b465a
F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15
F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa
F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c
F ext/fts3/fts3_snippet.c 19a906f8ed73ad8b670dfc271ceae7b3338c157e
F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a
F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef
F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894
F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68
F ext/fts3/fts3_write.c ffe13acc3867ea6b0fc8b9cfbf904bfae64eac84
F ext/fts3/fts3_write.c f2545f59a4cc2eb6739acb3d026b8a91a1f3d429
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -459,7 +459,7 @@ F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d
F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
F test/fts3ao.test 60a15590d3c8578e943e4a149524b16b9bc1be92
F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9
F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494
F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
@ -471,23 +471,25 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32
F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
F test/fts3d.test bf640d79722b720fa1c81834c48cdaa45d531b1a
F test/fts3defer.test ffd4e07f79a09660d4b3e2613b041ab9b6100d91
F test/fts3defer.test 2ea3fa028f8d9523f9c33dd8acc4555d567ea4ac
F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246
F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d
F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b
F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f
F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53
F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be
F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6
F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676
F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1
F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
F test/fts3sort.test 9a5176c9317bb545ec5f144d62e6fedb4da6c66e
F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
F test/fts4content.test c5f531ecfc3d446b90032cae212549dbbb18dd78
F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@ -622,7 +624,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0
F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31
F test/permutations.test 522823b47238cb1754198f80817fe9f9158ede55
F test/pragma.test 1ea0c85be853135bb7468e6eed48ee12b04794d4
F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@ -970,7 +972,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
P 54aecd929867606d14a062b501abbfb6f5f05e37
R dccf2fe03db8f585fac4df0bc3670756
U drh
Z aaf98f5abdd914fb531f49ab6760c2d4
P d65f63531c3f8e3e55e656f049240714a3d7433f df36ac948179f37b432a88701b6c79299e073ce8
R 3877ede50d6923a62a70ae0714da94fb
U dan
Z e6c406cc06dcce98e86f6b11f422dc50

2
manifest.uuid
View File

@ -1 +1 @@
d65f63531c3f8e3e55e656f049240714a3d7433f
8a4077057ddeb08e8edc5f20a75abaaba7a278ba

3
test/fts3ao.test
View File

@ -200,6 +200,9 @@ do_test fts3ao-4.7 {
SELECT * FROM t5;
}
} {{the quick brown fox} {jumped over the} {lazy dog}}
do_execsql_test fts3ao-4.8 {
SELECT snippet(t5, '[', ']') FROM t5 WHERE t5 MATCH 'the'
} {{[the] quick brown fox} {jumped over [the]}}
# Test that it is possible to rename an FTS4 table. Renaming an FTS4 table
# involves renaming the extra %_docsize and %_stat tables.

12
test/fts3defer.test
View File

@ -426,6 +426,18 @@ foreach {tn setup} {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
}
if {$tn>1} {
# These tests will not work with $tn==1, as in this case table t1 is
# created using FTS3. The ^ syntax is only available with FTS4 tables.
#
do_select_test 7.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw'
} {56 62}
do_select_test 7.2 {
SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm'
} {43}
}
}
set testprefix fts3defer

49
test/fts3fault2.test
View File

@ -82,4 +82,53 @@ do_faultsim_test 2.1 -prep {
faultsim_test_result {0 {a * 1 1 a 0 1 1 b * 1 1 b 0 1 1 c * 1 1 c 0 1 1 x * 1 1 x 1 1 1 y * 1 1 y 1 1 1 z * 1 1 z 1 1 1}}
}
do_faultsim_test 3.0 -faults oom* -prep {
faultsim_delete_and_reopen
db eval { CREATE TABLE 'xx yy'(a, b); }
} -body {
execsql {
CREATE VIRTUAL TABLE tt USING fts4(content="xx yy");
}
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 3.1 -faults oom* -prep {
faultsim_delete_and_reopen
db func zip zip
db func unzip unzip
} -body {
execsql {
CREATE VIRTUAL TABLE tt USING fts4(compress=zip, uncompress=unzip);
}
} -test {
faultsim_test_result {0 {}}
}
do_test 4.0 {
faultsim_delete_and_reopen
execsql {
CREATE VIRTUAL TABLE ft USING fts4(a, b);
INSERT INTO ft VALUES('U U T C O', 'F N D E S');
INSERT INTO ft VALUES('P H X G B', 'I D M R U');
INSERT INTO ft VALUES('P P X D M', 'Y V N T C');
INSERT INTO ft VALUES('Z L Q O W', 'D F U N Q');
INSERT INTO ft VALUES('A J D U P', 'C H M Q E');
INSERT INTO ft VALUES('P S A O H', 'S Z C W D');
INSERT INTO ft VALUES('T B N L W', 'C A K T I');
INSERT INTO ft VALUES('K E Z L O', 'L L Y C E');
INSERT INTO ft VALUES('C R E S V', 'Q V F W P');
INSERT INTO ft VALUES('S K H G W', 'R W Q F G');
}
faultsim_save_and_close
} {}
do_faultsim_test 4.1 -prep {
faultsim_restore_and_reopen
db eval {SELECT * FROM sqlite_master}
} -body {
execsql { INSERT INTO ft(ft) VALUES('rebuild') }
} -test {
faultsim_test_result {0 {}}
}
finish_test

163
test/fts3first.test Normal file
View File

@ -0,0 +1,163 @@
# 2011 October 18
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
ifcapable !fts3 {
finish_test
return
}
set testprefix fts3first
proc lreverse {L} {
set res [list]
for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
lappend res [lindex $L $ii]
}
set res
}
proc mit {blob} {
set scan(littleEndian) i*
set scan(bigEndian) I*
binary scan $blob $scan($::tcl_platform(byteOrder)) r
return $r
}
db func mit mit
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q');
INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y');
INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F');
INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');
CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
}
# Test queries.
#
foreach x {1 2} {
foreach {tn match res} {
1 "^K" {0 1 14}
2 "^S" {0 1 14}
3 "^W" {9 15 17}
4 "^J" {}
5 "^E" {12}
6 "V ^-E" {0 3 4 6 7 9 17 19}
7 "V -^E" {0 3 4 6 7 9 17 19}
8 "^-E V" {0 3 4 6 7 9 17 19}
9 "-^E V" {0 3 4 6 7 9 17 19}
10 "V" {0 3 4 6 7 9 12 17 19}
11 {"^K H"} {0 14}
12 {"K H"} {0 10 14}
13 {"K ^H"} {}
} {
set rev [lreverse $res]
do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
}
do_execsql_test 1.$x.[expr $tn+1] {
INSERT INTO x1(x1) VALUES('optimize');
INSERT INTO x2(x2) VALUES('optimize');
} {}
}
# Test the snippet() function.
#
foreach {tn match res} {
1 {^K} {{[K] H D S T} {[K] N J L W} {[K] H A B L}}
2 {^X} {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}}
3 {^X Y} {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}}
} {
set rev [lreverse $res]
do_execsql_test 1.3.$tn.1 {
SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match
} $res
do_execsql_test 1.3.$tn.2 {
SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match
} $rev
}
# Test matchinfo().
#
foreach {tn match res} {
1 {^K} {
{1 3 3 0 0 0 0 0 0}
{1 3 3 0 0 0 0 0 0}
{1 3 3 0 0 0 0 0 0}
}
2 {^X} {
{0 1 1 0 1 1 1 2 2}
{0 1 1 1 1 1 0 2 2}
{1 1 1 0 1 1 1 2 2}
}
3 {^X Y} {
{0 1 1 0 1 1 1 2 2 0 6 5 0 5 4 1 4 4}
{0 1 1 1 1 1 0 2 2 1 6 5 0 5 4 0 4 4}
{1 1 1 0 1 1 1 2 2 1 6 5 0 5 4 1 4 4}
}
} {
set rev [lreverse $res]
do_execsql_test 1.3.$tn.1 {
SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match
} $res
do_execsql_test 1.3.$tn.2 {
SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match
} $rev
}
# Test that ^ is ignored for FTS3 tables.
#
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE x3 USING fts3;
INSERT INTO x3 VALUES('A B C');
INSERT INTO x3 VALUES('B A C');
CREATE VIRTUAL TABLE x4 USING fts4;
INSERT INTO x4 VALUES('A B C');
INSERT INTO x4 VALUES('B A C');
}
do_execsql_test 2.2.1 {
SELECT * FROM x3 WHERE x3 MATCH '^A';
} {{A B C} {B A C}}
do_execsql_test 2.2.2 {
SELECT * FROM x4 WHERE x4 MATCH '^A';
} {{A B C}}
finish_test

1
test/fts3malloc.test
View File

@ -294,6 +294,7 @@ do_write_test fts3_malloc-5.1 ft_content {
do_test fts3_malloc-5.2 {
execsql { CREATE VIRTUAL TABLE ft8 USING fts3(x, tokenize porter) }
} {}
do_write_test fts3_malloc-5.3 ft_content {
INSERT INTO ft8 VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken')
}

16
test/fts3matchinfo.test
View File

@ -19,6 +19,7 @@ source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
set testprefix fts3matchinfo
set sqlite_fts3_enable_parentheses 0
proc mit {blob} {
set scan(littleEndian) i*
@ -57,6 +58,9 @@ do_catchsql_test 2.0 {
do_catchsql_test 2.1 {
CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3);
} {1 {unrecognized parameter: mtchinfo=fts3}}
do_catchsql_test 2.2 {
CREATE VIRTUAL TABLE x2 USING fts4(matchinfo=fts5);
} {1 {unrecognized matchinfo: fts5}}
# Check that with fts3, the "=" character is permitted in column definitions.
#
@ -224,6 +228,18 @@ do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} }
do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
x {
{0 1 1 1 1 1 0 0 0 0 0 0}
{1 1 1 0 1 1 0 0 0 0 0 0}
}
}
do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f -abcd'} {
x {
{0 1 1 1 1 1}
{1 1 1 0 1 1}
}
}
do_execsql_test 4.2.0 {
CREATE VIRTUAL TABLE t5 USING fts4;

10
test/fts3prefix.test
View File

@ -200,4 +200,14 @@ do_execsql_test 4.6 {
SELECT * FROM t3 WHERE t3 MATCH 'one*'
} {{one two three}}
#-------------------------------------------------------------------------
# Syntax tests.
#
do_catchsql_test 5.1 {
CREATE VIRTUAL TABLE t4 USING fts4(prefix="abc");
} {1 {error parsing prefix parameter: abc}}
do_catchsql_test 5.2 {
CREATE VIRTUAL TABLE t4 USING fts4(prefix="");
} {0 {}}
finish_test

5
test/fts3sort.test
View File

@ -138,6 +138,8 @@ foreach {tn param res} {
3 "order=dec" {1 {unrecognized order: dec}}
4 "order=xxx, order=asc" {1 {unrecognized order: xxx}}
5 "order=desc, order=asc" {0 {}}
6 "order=xxxx, order=asc" {1 {unrecognized order: xxxx}}
7 "order=desk" {1 {unrecognized order: desk}}
} {
execsql { DROP TABLE IF EXISTS t1 }
do_catchsql_test 2.1.$tn "
@ -157,6 +159,9 @@ do_execsql_test 2.2 {
do_execsql_test 2.3 {
SELECT docid FROM t2 WHERE t2 MATCH 'aa';
} {3 1}
do_execsql_test 2.4 {
SELECT docid FROM t2 WHERE t2 MATCH 'aa' ORDER BY content;
} {1 3}
#-------------------------------------------------------------------------
# Test that ticket [56be976859] has been fixed.

478
test/fts4content.test Normal file
View File

@ -0,0 +1,478 @@
# 2011 October 03
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the content=xxx FTS4 option.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts4content
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
#-------------------------------------------------------------------------
# Test organization:
#
# 1.* - Warm-body tests.
#
# 2.* - Querying a content=xxx FTS table.
#
# 3.* - Writing to a content=xxx FTS table.
#
# 4.* - The "INSERT INTO fts(fts) VALUES('rebuild')" command.
#
# 5.* - Check that CREATE TABLE, DROP TABLE and ALTER TABLE correctly
# ignore any %_content table when used with the content=xxx option.
#
# 6.* - Test the effects of messing with the schema of table xxx after
# creating a content=xxx FTS index.
#
do_execsql_test 1.1.1 {
CREATE TABLE t1(a, b, c);
INSERT INTO t1 VALUES('w x', 'x y', 'y z');
CREATE VIRTUAL TABLE ft1 USING fts4(content=t1);
}
do_execsql_test 1.1.2 {
PRAGMA table_info(ft1);
} {
0 a {} 0 {} 0
1 b {} 0 {} 0
2 c {} 0 {} 0
}
do_execsql_test 1.1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1}
do_execsql_test 1.1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}}
do_execsql_test 1.1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {}
do_execsql_test 1.1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1}
do_execsql_test 1.1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {}
do_execsql_test 1.2.1 {
DROP TABLE ft1;
CREATE VIRTUAL TABLE ft1 USING fts4(content=t1, b);
PRAGMA table_info(ft1);
} {
0 b {} 0 {} 0
}
do_execsql_test 1.2.2 {
SELECT *, rowid FROM ft1
} {{x y} 1}
#-------------------------------------------------------------------------
# The following block of tests - 2.* - test that a content=xxx FTS table
# can be queried. Also tested are cases where rows identified in the FTS
# are missing from the content table, and cases where the index is
# inconsistent with the content table.
#
do_execsql_test 2.0 {
CREATE TABLE t2(x);
INSERT INTO t2 VALUES('O S W W F U C R Q I C N P Z Y Y E Y Y E'); -- 1
INSERT INTO t2 VALUES('Y X U V L B E H Y J C Y A I A P V F V K'); -- 2
INSERT INTO t2 VALUES('P W I N J H I I N I F B K D U Q B Z S F'); -- 3
INSERT INTO t2 VALUES('N R O R H J R H G M D I U U B O M P A U'); -- 4
INSERT INTO t2 VALUES('Y O V O G T P N G T N F I V B U M J M G'); -- 5
INSERT INTO t2 VALUES('J O B N K N E C H Z R K J O U G M K L S'); -- 6
INSERT INTO t2 VALUES('S Z S R I Q U A P W R X H K C Z U L S P'); -- 7
INSERT INTO t2 VALUES('J C H N R C K R V N M O F Z M Z A I H W'); -- 8
INSERT INTO t2 VALUES('O Y G I S J U U W O D Z F J K N R P R L'); -- 9
INSERT INTO t2 VALUES('B G L K U R U P V X Z I H V R W C Q A S'); -- 10
INSERT INTO t2 VALUES('T F T J F F Y V F W N X K Q A Y L X W G'); -- 11
INSERT INTO t2 VALUES('C J U H B Q X L C M M Y E G V F W V Z C'); -- 12
INSERT INTO t2 VALUES('B W L T F S G X D P H N G M R I O A X I'); -- 13
INSERT INTO t2 VALUES('N G Y O K Q K Z N M H U J E D H U W R K'); -- 14
INSERT INTO t2 VALUES('U D T R U Y F J D S J X E H Q G V A S Z'); -- 15
INSERT INTO t2 VALUES('M I W P J S H R J D Q I C G P C T P H R'); -- 16
INSERT INTO t2 VALUES('J M N I S L X Q C A B F C B Y D H V R J'); -- 17
INSERT INTO t2 VALUES('F V Z W J Q L P X Y E W B U Q N H X K T'); -- 18
INSERT INTO t2 VALUES('R F S R Y O F Q E I E G H C B H R X Y N'); -- 19
INSERT INTO t2 VALUES('U Q Q Q T E P D M F X P J G H X C Q D L'); -- 20
}
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE ft2 USING fts4(content=t2);
INSERT INTO ft2(ft2) VALUES('rebuild');
-- Modify the backing table a bit: Row 17 is missing and the contents
-- of row 20 do not match the FTS index contents.
DELETE FROM t2 WHERE rowid = 17;
UPDATE t2 SET x = 'a b c d e f g h i j' WHERE rowid = 20;
}
foreach {tn match rowidlist} {
1 {S} {1 3 6 7 9 10 13 15 16 17 19}
2 {"S R"} {7 19}
3 {"N K N"} {6}
4 {"Q Q"} {20}
5 {"B Y D"} {17}
} {
do_execsql_test 2.2.1.$tn {
SELECT rowid FROM ft2 WHERE ft2 MATCH $match
} $rowidlist
do_execsql_test 2.2.2.$tn {
SELECT docid FROM ft2 WHERE ft2 MATCH $match
} $rowidlist
}
foreach {tn match result} {
1 {"N K N"} {{J O B N K N E C H Z R K J O U G M K L S}}
2 {"Q Q"} {{a b c d e f g h i j}}
3 {"B Y D"} {{}}
} {
do_execsql_test 2.3.$tn {
SELECT * FROM ft2 WHERE ft2 MATCH $match
} $result
}
foreach {tn match result} {
1 {"N K N"} {{..O B [N] [K] [N] E..}}
2 {"B Y D"} {{}}
3 {"Q Q"} {{a [b] [c] [d] e f..}}
} {
do_execsql_test 2.4.$tn {
SELECT snippet(ft2, '[', ']', '..', -1, 6) FROM ft2 WHERE ft2 MATCH $match
} $result
}
foreach {tn match result} {
1 {"N K N"} {{0 0 6 1 0 1 8 1 0 2 10 1}}
2 {"B Y D"} {{}}
3 {"Q Q"} {{0 0 2 1 0 0 4 1 0 1 4 1 0 1 6 1}}
4 {"Q D L"} {{}}
} {
do_execsql_test 2.5.$tn {
SELECT offsets(ft2) FROM ft2 WHERE ft2 MATCH $match
} $result
}
#-------------------------------------------------------------------------
# The following block of tests - 3.* - test that the FTS index can be
# modified by writing to the table. But that this has no effect on the
# content table.
#
do_execsql_test 3.1 {
CREATE TABLE t3(x, y);
CREATE VIRTUAL TABLE ft3 USING fts4(content=t3);
}
do_catchsql_test 3.1.1 {
INSERT INTO ft3 VALUES('a b c', 'd e f');
} {1 {constraint failed}}
do_execsql_test 3.1.2 {
INSERT INTO ft3(docid, x, y) VALUES(21, 'a b c', 'd e f');
SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
} {21}
do_execsql_test 3.1.3 { SELECT * FROM t3 } {}
# This DELETE does not work, since there is no row in [t3] to base the
# DELETE on. So the SELECT on [ft3] still returns rowid 21.
do_execsql_test 3.1.4 {
DELETE FROM ft3;
SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
} {21}
# If the row is added to [t3] before the DELETE on [ft3], it works.
do_execsql_test 3.1.5 {
INSERT INTO t3(rowid, x, y) VALUES(21, 'a b c', 'd e f');
DELETE FROM ft3;
SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
} {}
do_execsql_test 3.1.6 { SELECT rowid FROM t3 } {21}
do_execsql_test 3.2.1 {
INSERT INTO ft3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H');
INSERT INTO ft3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K');
INSERT INTO ft3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C');
INSERT INTO ft3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C');
INSERT INTO ft3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C');
INSERT INTO ft3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y');
INSERT INTO ft3(rowid, x, y) VALUES(6, 'I Q I S P', 'D R O Q B');
INSERT INTO ft3(rowid, x, y) VALUES(7, 'T K T Z J', 'B W D G O');
INSERT INTO ft3(rowid, x, y) VALUES(8, 'Y K F X T', 'D F G V G');
INSERT INTO ft3(rowid, x, y) VALUES(9, 'E L E T L', 'P W N F Z');
INSERT INTO ft3(rowid, x, y) VALUES(10, 'O G J G X', 'G J F E P');
INSERT INTO ft3(rowid, x, y) VALUES(11, 'O L N N Z', 'K E Z F D');
INSERT INTO ft3(rowid, x, y) VALUES(12, 'R Z M R J', 'X G I M Z');
INSERT INTO ft3(rowid, x, y) VALUES(13, 'L X N N X', 'R R N S T');
INSERT INTO ft3(rowid, x, y) VALUES(14, 'F L B J H', 'K W F L C');
INSERT INTO ft3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U');
INSERT INTO ft3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K');
INSERT INTO ft3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N');
INSERT INTO ft3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V');
INSERT INTO ft3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O');
}
foreach {tn match rowidlist} {
1 "N A" {5 19}
2 "x:O" {1 2 10 11 17}
3 "y:O" {0 2 6 7 18 19}
} {
set res [list]
foreach rowid $rowidlist { lappend res $rowid {} {} }
do_execsql_test 3.2.2.$tn {
SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
} $res
do_execsql_test 3.2.3.$tn {
SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
} $res
}
do_execsql_test 3.3.1 {
INSERT INTO t3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H');
INSERT INTO t3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K');
INSERT INTO t3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C');
INSERT INTO t3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C');
INSERT INTO t3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C');
INSERT INTO t3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y');
UPDATE ft3 SET x = y, y = x;
DELETE FROM t3;
}
foreach {tn match rowidlist} {
1 "N A" {5 19}
2 "x:O" {0 2 10 11 17}
3 "y:O" {1 2 6 7 18 19}
} {
set res [list]
foreach rowid $rowidlist { lappend res $rowid {} {} }
do_execsql_test 3.3.2.$tn {
SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
} $res
do_execsql_test 3.3.3.$tn {
SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
} $res
}
do_execsql_test 3.3.1 {
INSERT INTO t3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U');
INSERT INTO t3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K');
INSERT INTO t3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N');
INSERT INTO t3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V');
INSERT INTO t3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O');
DELETE FROM ft3;
}
foreach {tn match rowidlist} {
1 "N A" {5}
2 "x:O" {0 2 10 11}
3 "y:O" {1 2 6 7}
} {
set res [list]
foreach rowid $rowidlist { lappend res $rowid {} {} }
do_execsql_test 3.3.2.$tn {
SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
} $res
do_execsql_test 3.3.3.$tn {
SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
} $res
}
#-------------------------------------------------------------------------
# Test cases 4.* test the 'rebuild' command. On content=xxx and regular
# FTS tables.
#
do_execsql_test 4.0 {
CREATE TABLE t4(x);
CREATE VIRTUAL TABLE ft4 USING fts4(content=t4);
CREATE VIRTUAL TABLE ft4x USING fts4(x);
}
do_execsql_test 4.1.1 {
INSERT INTO ft4x(ft4x) VALUES('rebuild');
INSERT INTO ft4(ft4) VALUES('rebuild');
} {}
do_execsql_test 4.1.2 {
SELECT id, quote(value) FROM ft4_stat
} {0 X'000000'}
do_execsql_test 4.1.3 {
SELECT id, quote(value) FROM ft4x_stat
} {0 X'000000'}
do_execsql_test 4.2.1 {
INSERT INTO ft4x VALUES('M G M F T');
INSERT INTO ft4x VALUES('Z Q C A U');
INSERT INTO ft4x VALUES('N L L V');
INSERT INTO ft4x VALUES('T F D X D');
INSERT INTO ft4x VALUES('Z H I S D');
SELECT id, quote(value) FROM ft4x_stat
} {0 X'05182B'}
do_execsql_test 4.2.2 {
INSERT INTO ft4(rowid, x) SELECT rowid, * FROM ft4x;
SELECT id, quote(value) FROM ft4_stat
} {0 X'05182B'}
do_execsql_test 4.2.3 {
SELECT docid, quote(size) FROM ft4_docsize
} {1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}
do_execsql_test 4.2.4 {
INSERT INTO ft4x(ft4x) VALUES('rebuild');
SELECT id, quote(value) FROM ft4x_stat;
SELECT docid, quote(size) FROM ft4x_docsize
} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}
do_execsql_test 4.2.5 {
INSERT INTO ft4(ft4) VALUES('rebuild');
SELECT id, quote(value) FROM ft4_stat;
SELECT docid, quote(size) FROM ft4_docsize
} {0 X'000000'}
do_execsql_test 4.2.6 {
INSERT INTO t4(rowid, x) SELECT rowid, x FROM ft4x;
INSERT INTO ft4(ft4) VALUES('rebuild');
SELECT id, quote(value) FROM ft4_stat;
SELECT docid, quote(size) FROM ft4_docsize
} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}
#-------------------------------------------------------------------------
# Test cases 5.* test that the following commands do not create/move or
# delete a %_content table when used with a content=xxx FTS table.
#
do_execsql_test 5.1.1 {
CREATE TABLE t5(a, b, c, d);
CREATE VIRTUAL TABLE ft5 USING fts4(content=t5);
SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
} {
t5 ft5 ft5_segments ft5_segdir
sqlite_autoindex_ft5_segdir_1 ft5_docsize ft5_stat
}
do_execsql_test 5.1.2 {
ALTER TABLE ft5 RENAME TO ft6;
SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
} {
t5
}
do_execsql_test 5.1.3 {
SELECT name FROM sqlite_master WHERE name LIKE '%t6%';
} {
ft6 ft6_segments ft6_segdir
sqlite_autoindex_ft6_segdir_1 ft6_docsize ft6_stat
}
do_execsql_test 5.1.4 {
INSERT INTO t5 VALUES('a', 'b', 'c', 'd');
INSERT INTO ft6(ft6) VALUES('rebuild');
SELECT rowid FROM ft6 WHERE ft6 MATCH 'b';
} {1}
do_execsql_test 5.1.5 {
DROP TABLE ft6;
SELECT * FROM t5;
} {a b c d}
do_execsql_test 5.1.6 {
SELECT name FROM sqlite_master WHERE name LIKE '%t6%';
} {
}
do_execsql_test 5.1.7 {
CREATE VIRTUAL TABLE ft5 USING fts4(content=t5);
CREATE TABLE t5_content(a, b);
DROP TABLE ft5;
SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
} {
t5 t5_content
}
#-------------------------------------------------------------------------
# Test cases 6.* test
#
do_catchsql_test 6.1.1 {
CREATE VIRTUAL TABLE ft7 USING fts4(content=t7);
} {1 {vtable constructor failed: ft7}}
do_execsql_test 6.2.1 {
CREATE TABLE t7(one, two);
CREATE VIRTUAL TABLE ft7 USING fts4(content=t7);
INSERT INTO t7 VALUES('A B', 'B A');
INSERT INTO t7 VALUES('C D', 'A A');
SELECT * FROM ft7;
} {
{A B} {B A} {C D} {A A}
}
do_catchsql_test 6.2.2 {
DROP TABLE t7;
SELECT * FROM ft7;
} {1 {SQL logic error or missing database}}
db close
sqlite3 db test.db
do_execsql_test 6.2.3 {
SELECT name FROM sqlite_master WHERE name LIKE '%t7%'
} {
ft7 ft7_segments ft7_segdir sqlite_autoindex_ft7_segdir_1
ft7_docsize ft7_stat
}
do_catchsql_test 6.2.4 {
SELECT * FROM ft7;
} {1 {vtable constructor failed: ft7}}
do_execsql_test 6.2.5 {
CREATE TABLE t7(x, y);
INSERT INTO t7 VALUES('A B', 'B A');
INSERT INTO t7 VALUES('C D', 'A A');
SELECT * FROM ft7;
} {
{A B} {B A} {C D} {A A}
}
do_execsql_test 6.2.6 {
INSERT INTO ft7(ft7) VALUES('rebuild');
SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"';
} {2}
do_execsql_test 6.2.7 {
DROP TABLE t7;
CREATE TABLE t7(x);
}
do_catchsql_test 6.2.8 {
SELECT * FROM ft7 WHERE ft7 MATCH '"A A"';
} {1 {SQL logic error or missing database}}
do_catchsql_test 6.2.9 {
SELECT * FROM ft7 WHERE ft7 MATCH '"A A"';
} {1 {SQL logic error or missing database}}
db close
sqlite3 db test.db
do_catchsql_test 6.2.10 {
SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"';
} {0 2}
do_catchsql_test 6.2.11 {
SELECT rowid, * FROM ft7 WHERE ft7 MATCH '"A A"';
} {0 {2 {}}}
#-------------------------------------------------------------------------
# Test cases 7.*
#
do_execsql_test 7.1.1 {
CREATE VIRTUAL TABLE ft8 USING fts4(content=nosuchtable, x);
INSERT INTO ft8(docid, x) VALUES(13, 'U O N X G');
INSERT INTO ft8(docid, x) VALUES(14, 'C J J U B');
INSERT INTO ft8(docid, x) VALUES(15, 'N J Y G X');
INSERT INTO ft8(docid, x) VALUES(16, 'R Y D O R');
INSERT INTO ft8(docid, x) VALUES(17, 'I Y T Q O');
}
do_execsql_test 7.1.2 {
SELECT docid FROM ft8 WHERE ft8 MATCH 'N';
} {13 15}
finish_test

6
test/permutations.test
View File

@ -180,10 +180,12 @@ test_suite "fts3" -prefix "" -description {
fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test
fts3near.test fts3query.test fts3shared.test fts3snippet.test
fts3sort.test
fts3fault.test fts3malloc.test fts3matchinfo.test
fts3aux1.test fts3comp1.test fts3auto.test
fts4aa.test fts4content.test
fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
fts3corrupt2.test
fts3first.test
}