mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add some code for an experimental fts5 module. Does not work yet.

Browse Source 
FossilOrigin-Name: 1e0648dcf283d4f1f6159db4d2433b6cc635992e
This commit is contained in:
dan 2014-06-23 11:33:22 +00:00
parent 685ffb134a
commit e0fa4107c2
13 changed files with 5460 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ext/fts3/fts3.h
View File

@ -20,6 +20,7 @@ extern "C" {
#endif /* __cplusplus */
int sqlite3Fts3Init(sqlite3 *db);
int sqlite3Fts5Init(sqlite3 *db);
#ifdef __cplusplus
} /* extern "C" */

425
ext/fts5/fts5.c Normal file
View File

@ -0,0 +1,425 @@
/*
** 2014 Jun 09
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/
#include "fts5Int.h"
typedef struct Fts5Table Fts5Table;
struct Fts5Table {
sqlite3_vtab base; /* Base class used by SQLite core */
Fts5Config *pConfig; /* Virtual table configuration */
Fts5Index *pIndex; /* Full-text index */
Fts5Storage *pStorage; /* Document store */
};
/*
** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
** argument is non-zero, attempt delete the shadow tables from teh database
*/
static int fts5FreeVtab(Fts5Table *pTab, int bDestroy){
int rc = SQLITE_OK;
if( pTab ){
int rc2;
rc2 = sqlite3Fts5IndexClose(pTab->pIndex, bDestroy);
if( rc==SQLITE_OK ) rc = rc2;
rc2 = sqlite3Fts5StorageClose(pTab->pStorage, bDestroy);
if( rc==SQLITE_OK ) rc = rc2;
sqlite3Fts5ConfigFree(pTab->pConfig);
sqlite3_free(pTab);
}
return rc;
}
/*
** The xDisconnect() virtual table method.
*/
static int fts5DisconnectMethod(sqlite3_vtab *pVtab){
return fts5FreeVtab((Fts5Table*)pVtab, 0);
}
/*
** The xDestroy() virtual table method.
*/
static int fts5DestroyMethod(sqlite3_vtab *pVtab){
return fts5FreeVtab((Fts5Table*)pVtab, 1);
}
/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
** argv[0] -> module name ("fts5")
** argv[1] -> database name
** argv[2] -> table name
** argv[...] -> "column name" and other module argument fields.
*/
static int fts5InitVtab(
int bCreate, /* True for xCreate, false for xConnect */
sqlite3 *db, /* The SQLite database connection */
void *pAux, /* Hash table containing tokenizers */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
char **pzErr /* Write any error message here */
){
int rc; /* Return code */
Fts5Config *pConfig; /* Results of parsing argc/argv */
Fts5Table *pTab = 0; /* New virtual table object */
/* Parse the arguments */
rc = sqlite3Fts5ConfigParse(db, argc, (const char**)argv, &pConfig, pzErr);
assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
/* Allocate the new vtab object */
if( rc==SQLITE_OK ){
pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table));
if( pTab==0 ){
rc = SQLITE_NOMEM;
}else{
memset(pTab, 0, sizeof(Fts5Table));
pTab->pConfig = pConfig;
}
}
/* Open the index sub-system */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr);
}
/* Open the storage sub-system */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5StorageOpen(
pConfig, pTab->pIndex, bCreate, &pTab->pStorage, pzErr
);
}
/* Call sqlite3_declare_vtab() */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
}
if( rc!=SQLITE_OK ){
fts5FreeVtab(pTab, 0);
pTab = 0;
}
*ppVTab = (sqlite3_vtab*)pTab;
return rc;
}
/*
** The xConnect() and xCreate() methods for the virtual table. All the
** work is done in function fts5InitVtab().
*/
static int fts5ConnectMethod(
sqlite3 *db, /* Database connection */
void *pAux, /* Pointer to tokenizer hash table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
}
static int fts5CreateMethod(
sqlite3 *db, /* Database connection */
void *pAux, /* Pointer to tokenizer hash table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
}
/*
** Implementation of the xBestIndex method for FTS3 tables. There
** are three possible strategies, in order of preference:
**
** 1. Direct lookup by rowid or docid.
** 2. Full-text search using a MATCH operator on a non-docid column.
** 3. Linear scan of %_content table.
*/
static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
return SQLITE_OK;
}
/*
** Implementation of xOpen method.
*/
static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
return SQLITE_OK;
}
/*
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
return SQLITE_OK;
}
/*
** Advance the cursor to the next row in the table that matches the
** search criteria.
**
** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
** even if we reach end-of-file. The fts5EofMethod() will be called
** subsequently to determine whether or not an EOF was hit.
*/
static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
return SQLITE_OK;
}
/*
** This is the xFilter interface for the virtual table. See
** the virtual table xFilter method documentation for additional
** information.
*/
static int fts5FilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
return SQLITE_OK;
}
/*
** This is the xEof method of the virtual table. SQLite calls this
** routine to find out if it has reached the end of a result set.
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
return 1;
}
/*
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5
** exposes %_content.docid as the rowid for the virtual table. The
** rowid should be written to *pRowid.
*/
static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
return SQLITE_OK;
}
/*
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
return SQLITE_OK;
}
/*
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
** INSERT INTO fts(fts) VALUES($pVal)
**
** Argument pVal is the value assigned to column "fts" by the INSERT
** statement. This function returns SQLITE_OK if successful, or an SQLite
** error code if an error occurs.
*/
static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
const char *z = sqlite3_value_text(pVal);
int n = sqlite3_value_bytes(pVal);
int rc = SQLITE_ERROR;
if( 0==sqlite3_stricmp("integrity-check", z) ){
rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
}else
if( n>5 && 0==sqlite3_strnicmp("pgsz=", z, 5) ){
int pgsz = atoi(&z[5]);
if( pgsz<32 ) pgsz = 32;
sqlite3Fts5IndexPgsz(pTab->pIndex, pgsz);
rc = SQLITE_OK;
}
return rc;
}
/*
** This function is the implementation of the xUpdate callback used by
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.
*/
static int fts5UpdateMethod(
sqlite3_vtab *pVtab, /* Virtual table handle */
int nArg, /* Size of argument array */
sqlite3_value **apVal, /* Array of arguments */
sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
){
Fts5Table *pTab = (Fts5Table*)pVtab;
Fts5Config *pConfig = pTab->pConfig;
int eType0; /* value_type() of apVal[0] */
int eConflict; /* ON CONFLICT for this DML */
int rc = SQLITE_OK; /* Return code */
assert( nArg==1 || nArg==(2 + pConfig->nCol + 1) );
if( SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
}
eType0 = sqlite3_value_type(apVal[0]);
eConflict = sqlite3_vtab_on_conflict(pConfig->db);
assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
if( eType0==SQLITE_INTEGER ){
i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
}
if( rc==SQLITE_OK && nArg>1 ){
rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
}
return rc;
}
/*
** Implementation of xSync() method.
*/
static int fts5SyncMethod(sqlite3_vtab *pVtab){
int rc;
Fts5Table *pTab = (Fts5Table*)pVtab;
rc = sqlite3Fts5IndexSync(pTab->pIndex);
return rc;
}
/*
** Implementation of xBegin() method.
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
return SQLITE_OK;
}
/*
** Implementation of xCommit() method. This is a no-op. The contents of
** the pending-terms hash-table have already been flushed into the database
** by fts5SyncMethod().
*/
static int fts5CommitMethod(sqlite3_vtab *pVtab){
return SQLITE_OK;
}
/*
** Implementation of xRollback(). Discard the contents of the pending-terms
** hash-table. Any changes made to the database are reverted by SQLite.
*/
static int fts5RollbackMethod(sqlite3_vtab *pVtab){
Fts5Table *pTab = (Fts5Table*)pVtab;
int rc;
rc = sqlite3Fts5IndexRollback(pTab->pIndex);
return rc;
}
/*
** This routine implements the xFindFunction method for the FTS3
** virtual table.
*/
static int fts5FindFunctionMethod(
sqlite3_vtab *pVtab, /* Virtual table handle */
int nArg, /* Number of SQL function arguments */
const char *zName, /* Name of SQL function */
void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
void **ppArg /* Unused */
){
/* No function of the specified name was found. Return 0. */
return 0;
}
/*
** Implementation of FTS3 xRename method. Rename an fts5 table.
*/
static int fts5RenameMethod(
sqlite3_vtab *pVtab, /* Virtual table handle */
const char *zName /* New name of table */
){
int rc = SQLITE_OK;
return rc;
}
/*
** The xSavepoint() method.
**
** Flush the contents of the pending-terms table to disk.
*/
static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
int rc = SQLITE_OK;
return rc;
}
/*
** The xRelease() method.
**
** This is a no-op.
*/
static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
return SQLITE_OK;
}
/*
** The xRollbackTo() method.
**
** Discard the contents of the pending terms table.
*/
static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
return SQLITE_OK;
}
static const sqlite3_module fts5Module = {
/* iVersion */ 2,
/* xCreate */ fts5CreateMethod,
/* xConnect */ fts5ConnectMethod,
/* xBestIndex */ fts5BestIndexMethod,
/* xDisconnect */ fts5DisconnectMethod,
/* xDestroy */ fts5DestroyMethod,
/* xOpen */ fts5OpenMethod,
/* xClose */ fts5CloseMethod,
/* xFilter */ fts5FilterMethod,
/* xNext */ fts5NextMethod,
/* xEof */ fts5EofMethod,
/* xColumn */ fts5ColumnMethod,
/* xRowid */ fts5RowidMethod,
/* xUpdate */ fts5UpdateMethod,
/* xBegin */ fts5BeginMethod,
/* xSync */ fts5SyncMethod,
/* xCommit */ fts5CommitMethod,
/* xRollback */ fts5RollbackMethod,
/* xFindFunction */ fts5FindFunctionMethod,
/* xRename */ fts5RenameMethod,
/* xSavepoint */ fts5SavepointMethod,
/* xRelease */ fts5ReleaseMethod,
/* xRollbackTo */ fts5RollbackToMethod,
};
int sqlite3Fts5Init(sqlite3 *db){
int rc;
rc = sqlite3_create_module_v2(db, "fts5", &fts5Module, 0, 0);
if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(db);
return rc;
}

310
ext/fts5/fts5Int.h Normal file
View File

@ -0,0 +1,310 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#ifndef _FTS5INT_H
#define _FTS5INT_H
#include "sqliteInt.h"
#include "fts3_tokenizer.h"
/*
** Maximum number of prefix indexes on single FTS5 table. This must be
** less than 32. If it is set to anything large than that, an #error
** directive in fts5_index.c will cause the build to fail.
*/
#define FTS5_MAX_PREFIX_INDEXES 31
#define FTS5_DEFAULT_NEARDIST 10
/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/
typedef struct Fts5Config Fts5Config;
/*
** An instance of the following structure encodes all information that can
** be gleaned from the CREATE VIRTUAL TABLE statement.
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
char *zDb; /* Database holding FTS index (e.g. "main") */
char *zName; /* Name of FTS index */
int nCol; /* Number of columns */
char **azCol; /* Column names */
int nPrefix; /* Number of prefix indexes */
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
sqlite3_tokenizer *pTokenizer; /* Tokenizer instance for this table */
};
int sqlite3Fts5ConfigParse(sqlite3*, int, const char**, Fts5Config**, char**);
void sqlite3Fts5ConfigFree(Fts5Config*);
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, const char*, int, int, int, int) /* Callback */
);
void sqlite3Fts5Dequote(char *z);
/*
** End of interface to code in fts5_config.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_index.c. fts5_index.c contains contains code
** to access the data stored in the %_data table.
*/
typedef struct Fts5Index Fts5Index;
typedef struct Fts5IndexIter Fts5IndexIter;
/*
** Values used as part of the flags argument passed to IndexQuery().
*/
#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
#define FTS5INDEX_QUERY_ASC 0x0002 /* Docs in ascending rowid order */
#define FTS5INDEX_QUERY_MATCH 0x0004 /* Use the iMatch arg to Next() */
#define FTS5INDEX_QUERY_DELETE 0x0008 /* Visit delete markers */
/*
** Create/destroy an Fts5Index object.
*/
int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy);
/*
** for(
** pIter = sqlite3Fts5IndexQuery(p, "token", 5, 0);
** 0==sqlite3Fts5IterEof(pIter);
** sqlite3Fts5IterNext(pIter)
** ){
** i64 iDocid = sqlite3Fts5IndexDocid(pIter);
** }
*/
/*
** Open a new iterator to iterate though all docids that match the
** specified token or token prefix.
*/
Fts5IndexIter *sqlite3Fts5IndexQuery(
Fts5Index *p, /* FTS index to query */
const char *pToken, int nToken, /* Token (or prefix) to query for */
int flags /* Mask of FTS5INDEX_QUERY_X flags */
);
/*
** Docid list iteration.
*/
int sqlite3Fts5IterEof(Fts5IndexIter*);
void sqlite3Fts5IterNext(Fts5IndexIter*, i64 iMatch);
int sqlite3Fts5IterSeek(Fts5IndexIter*, i64 iDocid);
i64 sqlite3Fts5IterDocid(Fts5IndexIter*);
/*
** Position list iteration.
**
** for(
** iPos=sqlite3Fts5IterFirstPos(pIter, iCol);
** iPos>=0;
** iPos=sqlite3Fts5IterNextPos(pIter)
** ){
** // token appears at position iPos of column iCol of the current document
** }
*/
int sqlite3Fts5IterFirstPos(Fts5IndexIter*, int iCol);
int sqlite3Fts5IterNextPos(Fts5IndexIter*);
/*
** Close an iterator opened by sqlite3Fts5IndexQuery().
*/
void sqlite3Fts5IterClose(Fts5IndexIter*);
/*
** Insert or remove data to or from the index. Each time a document is
** added to or removed from the index, this function is called one or more
** times.
**
** For an insert, it must be called once for each token in the new document.
** If the operation is a delete, it must be called (at least) once for each
** unique token in the document with an iCol value less than zero. The iPos
** argument is ignored for a delete.
*/
void sqlite3Fts5IndexWrite(
Fts5Index *p, /* Index to write to */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
const char *pToken, int nToken /* Token to add or remove to or from index */
);
/*
** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
** document iDocid.
*/
void sqlite3Fts5IndexBeginWrite(
Fts5Index *p, /* Index to write to */
i64 iDocid /* Docid to add or remove data from */
);
/*
** Flush any data stored in the in-memory hash tables to the database.
**
** This is called whenever (a) the main transaction is committed or (b) a
** new sub-transaction is opened.
*/
void sqlite3Fts5IndexFlush(Fts5Index *p);
int sqlite3Fts5IndexSync(Fts5Index *p);
/*
** Discard any data stored in the in-memory hash tables. Do not write it
** to the database. Additionally, assume that the contents of the %_data
** table may have changed on disk. So any in-memory caches of %_data
** records must be invalidated.
**
** This is called (a) whenever a main or sub-transaction is rolled back,
** and (b) whenever the read transaction is closed.
*/
int sqlite3Fts5IndexRollback(Fts5Index *p);
/*
** Retrieve and clear the current error code, respectively.
*/
int sqlite3Fts5IndexErrcode(Fts5Index*);
void sqlite3Fts5IndexReset(Fts5Index*);
/*
** Get (bSet==0) or set (bSet!=0) the "averages" record.
*/
void sqlite3Fts5IndexAverages(Fts5Index *p, int bSet, int nAvg, int *aAvg);
/*
** Functions called by the storage module as part of integrity-check.
*/
u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5IndexInit(sqlite3*);
void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_storage.c. fts5_storage.c contains contains
** code to access the data stored in the %_content and %_docsize tables.
*/
typedef struct Fts5Storage Fts5Storage;
int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy);
int sqlite3Fts5DropTable(Fts5Config*, const char *zPost);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, char **pzErr);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_expr.c.
*/
typedef struct Fts5Expr Fts5Expr;
typedef struct Fts5Parse Fts5Parse;
typedef struct Fts5Token Fts5Token;
typedef struct Fts5ExprPhrase Fts5ExprPhrase;
typedef struct Fts5ExprNearset Fts5ExprNearset;
struct Fts5Token {
const char *p; /* Token text (not NULL terminated) */
int n; /* Size of buffer p in bytes */
};
int sqlite3Fts5ExprNew(
Fts5Config *pConfig,
Fts5Index *pIdx,
const char *zExpr,
Fts5Expr **ppNew,
char **pzErr
);
int sqlite3Fts5ExprFirst(Fts5Expr *p);
int sqlite3Fts5ExprNext(Fts5Expr *p);
int sqlite3Fts5ExprEof(Fts5Expr *p);
i64 sqlite3Fts5ExprRowid(Fts5Expr *p);
void sqlite3Fts5ExprFree(Fts5Expr *p);
// int sqlite3Fts5IterFirstPos(Fts5Expr*, int iCol, int *piPos);
// int sqlite3Fts5IterNextPos(Fts5Expr*, int *piPos);
/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5ExprInit(sqlite3*);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y. */
void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
Fts5Expr *sqlite3Fts5ParseExpr(
Fts5Parse *pParse,
int eType,
Fts5Expr *pLeft,
Fts5Expr *pRight,
Fts5ExprNearset *pNear
);
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse,
Fts5ExprPhrase *pPhrase,
Fts5Token *pToken,
int bPrefix
);
Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5Parse*,
Fts5ExprNearset*,
Fts5ExprPhrase*
);
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5Expr *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
#endif

318
ext/fts5/fts5_config.c Normal file
View File

@ -0,0 +1,318 @@
/*
** 2014 Jun 09
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/
#include "fts5Int.h"
/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters. The conversion is done in-place. If the
** input does not begin with a quote character, then this routine
** is a no-op.
**
** Examples:
**
** "abc" becomes abc
** 'xyz' becomes xyz
** [pqr] becomes pqr
** `mno` becomes mno
*/
void sqlite3Fts5Dequote(char *z){
char quote; /* Quote character (if any ) */
quote = z[0];
if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
int iIn = 1; /* Index of next byte to read from input */
int iOut = 0; /* Index of next byte to write to output */
/* If the first byte was a '[', then the close-quote character is a ']' */
if( quote=='[' ) quote = ']';
while( ALWAYS(z[iIn]) ){
if( z[iIn]==quote ){
if( z[iIn+1]!=quote ) break;
z[iOut++] = quote;
iIn += 2;
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
}
}
/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
Fts5Config *pConfig, /* Configuration object to update */
char *zCmd, /* Special command to parse */
char *zArg, /* Argument to parse */
char **pzErr /* OUT: Error message */
){
if( sqlite3_stricmp(zCmd, "prefix")==0 ){
char *p;
if( pConfig->aPrefix ){
*pzErr = sqlite3_mprintf("multiple prefix=... directives");
return SQLITE_ERROR;
}
pConfig->aPrefix = sqlite3_malloc(sizeof(int) * FTS5_MAX_PREFIX_INDEXES);
p = zArg;
while( p[0] ){
int nPre = 0;
while( p[0]==' ' ) p++;
while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
nPre = nPre*10 + (p[0] - '0');
p++;
}
while( p[0]==' ' ) p++;
if( p[0]==',' ){
p++;
}else if( p[0] ){
*pzErr = sqlite3_mprintf("malformed prefix=... directive");
return SQLITE_ERROR;
}
if( nPre==0 || nPre>=1000 ){
*pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
return SQLITE_ERROR;
}
pConfig->aPrefix[pConfig->nPrefix] = nPre;
pConfig->nPrefix++;
}
return SQLITE_OK;
}
*pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
return SQLITE_ERROR;
}
/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(const char *z){
return sqlite3_mprintf("%s", z);
}
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module**);
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
sqlite3_tokenizer *pTokenizer; /* Tokenizer instance */
int rc; /* Return code */
sqlite3Fts3SimpleTokenizerModule(&pMod);
rc = pMod->xCreate(0, 0, &pTokenizer);
if( rc==SQLITE_OK ){
pTokenizer->pModule = pMod;
pConfig->pTokenizer = pTokenizer;
}
return rc;
}
/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate
** or xConnect method of the virtual table. This function attempts to
** allocate an instance of Fts5Config containing the results of parsing
** those arguments.
**
** If successful, SQLITE_OK is returned and *ppOut is set to point to the
** new Fts5Config object. If an error occurs, an SQLite error code is
** returned, *ppOut is set to NULL and an error message may be left in
** *pzErr. It is the responsibility of the caller to eventually free any
** such error message using sqlite3_free().
*/
int sqlite3Fts5ConfigParse(
sqlite3 *db,
int nArg, /* Number of arguments */
const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
Fts5Config **ppOut, /* OUT: Results of parse */
char **pzErr /* OUT: Error message */
){
int rc = SQLITE_OK; /* Return code */
Fts5Config *pRet; /* New object to return */
*ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
if( pRet==0 ) return SQLITE_NOMEM;
memset(pRet, 0, sizeof(Fts5Config));
pRet->db = db;
pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
pRet->zDb = fts5Strdup(azArg[1]);
pRet->zName = fts5Strdup(azArg[2]);
if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
for(i=3; rc==SQLITE_OK && i<nArg; i++){
char *zDup = fts5Strdup(azArg[i]);
if( zDup==0 ){
rc = SQLITE_NOMEM;
}else{
/* Check if this is a special directive - "cmd=arg" */
if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
char *p = zDup;
while( *p && *p!='=' ) p++;
if( *p ){
char *zArg = &p[1];
*p = '\0';
sqlite3Fts5Dequote(zArg);
rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
sqlite3_free(zDup);
zDup = 0;
}
}
/* If it is not a special directive, it must be a column name */
if( zDup ){
sqlite3Fts5Dequote(zDup);
pRet->azCol[pRet->nCol++] = zDup;
}
}
}
}
if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
rc = fts5ConfigDefaultTokenizer(pRet);
}
if( rc!=SQLITE_OK ){
sqlite3Fts5ConfigFree(pRet);
*ppOut = 0;
}
return rc;
}
/*
** Free the configuration object passed as the only argument.
*/
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
if( pConfig->pTokenizer ){
pConfig->pTokenizer->pModule->xDestroy(pConfig->pTokenizer);
}
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; i<pConfig->nCol; i++){
sqlite3_free(pConfig->azCol[i]);
}
sqlite3_free(pConfig->azCol);
sqlite3_free(pConfig->aPrefix);
sqlite3_free(pConfig);
}
}
/*
** Call sqlite3_declare_vtab() based on the contents of the configuration
** object passed as the only argument. Return SQLITE_OK if successful, or
** an SQLite error code if an error occurs.
*/
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
int i;
int rc;
char *zSql;
char *zOld;
zSql = (char*)sqlite3_mprintf("CREATE TABLE x(");
for(i=0; zSql && i<pConfig->nCol; i++){
zOld = zSql;
zSql = sqlite3_mprintf("%s%s%Q", zOld, (i==0?"":", "), pConfig->azCol[i]);
sqlite3_free(zOld);
}
if( zSql ){
zOld = zSql;
zSql = sqlite3_mprintf("%s, %Q HIDDEN)", zOld, pConfig->zName);
sqlite3_free(zOld);
}
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_declare_vtab(pConfig->db, zSql);
sqlite3_free(zSql);
}
return rc;
}
/*
** Tokenize the text passed via the second and third arguments.
**
** The callback is invoked once for each token in the input text. The
** arguments passed to it are, in order:
**
** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
** const char *pToken // Pointer to buffer containing token
** int nToken // Size of token in bytes
** int iStart // Byte offset of start of token within input text
** int iEnd // Byte offset of end of token within input text
** int iPos // Position of token in input (first token is 0)
**
** If the callback returns a non-zero value the tokenization is abandoned
** and no further callbacks are issued.
**
** This function returns SQLITE_OK if successful or an SQLite error code
** if an error occurs. If the tokenization was abandoned early because
** the callback returned SQLITE_DONE, this is not an error and this function
** still returns SQLITE_OK. Or, if the tokenization was abandoned early
** because the callback returned another non-zero value, it is assumed
** to be an SQLite error code and returned to the caller.
*/
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, const char*, int, int, int, int) /* Callback */
){
const sqlite3_tokenizer_module *pMod = pConfig->pTokenizer->pModule;
sqlite3_tokenizer_cursor *pCsr = 0;
int rc;
rc = pMod->xOpen(pConfig->pTokenizer, pText, nText, &pCsr);
assert( rc==SQLITE_OK || pCsr==0 );
if( rc==SQLITE_OK ){
const char *pToken; /* Pointer to token buffer */
int nToken; /* Size of token in bytes */
int iStart, iEnd, iPos; /* Start, end and position of token */
pCsr->pTokenizer = pConfig->pTokenizer;
for(rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos);
rc==SQLITE_OK;
rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos)
){
if( (rc = xToken(pCtx, pToken, nToken, iStart, iEnd, iPos)) ) break;
}
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
pMod->xClose(pCsr);
}
return rc;
}

670
ext/fts5/fts5_expr.c Normal file
View File

@ -0,0 +1,670 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#include "fts5Int.h"
#include "fts5parse.h"
/*
** All token types in the generated fts5parse.h file are greater than 0.
*/
#define FTS5_EOF 0
typedef struct Fts5ExprTerm Fts5ExprTerm;
/*
** Functions generated by lemon from fts5parse.y.
*/
void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(size_t));
void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*));
void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
/*
** eType:
** Expression node type. Always one of:
**
** FTS5_AND (pLeft, pRight valid)
** FTS5_OR (pLeft, pRight valid)
** FTS5_NOT (pLeft, pRight valid)
** FTS5_STRING (pNear valid)
*/
struct Fts5Expr {
int eType; /* Node type */
Fts5Expr *pLeft; /* Left hand child node */
Fts5Expr *pRight; /* Right hand child node */
Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */
};
/*
** An instance of the following structure represents a single search term
** or term prefix.
*/
struct Fts5ExprTerm {
int bPrefix; /* True for a prefix term */
char *zTerm; /* nul-terminated term */
};
/*
** A phrase. One or more terms that must appear in a contiguous sequence
** within a document for it to match.
*/
struct Fts5ExprPhrase {
int nTerm; /* Number of entries in aTerm[] */
Fts5ExprTerm aTerm[0]; /* Terms that make up this phrase */
};
/*
** One or more phrases that must appear within a certain token distance of
** each other within each matching document.
*/
struct Fts5ExprNearset {
int nNear; /* NEAR parameter */
int iCol; /* Column to search (-1 -> all columns) */
int nPhrase; /* Number of entries in aPhrase[] array */
Fts5ExprPhrase *apPhrase[0]; /* Array of phrase pointers */
};
/*
** Parse context.
*/
struct Fts5Parse {
Fts5Config *pConfig;
char *zErr;
int rc;
Fts5Expr *pExpr; /* Result of a successful parse */
};
void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
if( pParse->rc==SQLITE_OK ){
va_list ap;
va_start(ap, zFmt);
pParse->zErr = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
pParse->rc = SQLITE_ERROR;
}
}
static int fts5ExprIsspace(char t){
return t==' ' || t=='\t' || t=='\n' || t=='\r';
}
static int fts5ExprIstoken(char t){
return fts5ExprIsspace(t)==0 && t!='\0'
&& t!=':' && t!='(' && t!=')'
&& t!=',' && t!='+' && t!='*';
}
/*
** Read the first token from the nul-terminated string at *pz.
*/
static int fts5ExprGetToken(
Fts5Parse *pParse,
const char **pz, /* IN/OUT: Pointer into buffer */
Fts5Token *pToken
){
const char *z = *pz;
int tok;
/* Skip past any whitespace */
while( fts5ExprIsspace(*z) ) z++;
pToken->p = z;
pToken->n = 1;
switch( *z ){
case '(': tok = FTS5_LP; break;
case ')': tok = FTS5_RP; break;
case ':': tok = FTS5_COLON; break;
case ',': tok = FTS5_COMMA; break;
case '+': tok = FTS5_PLUS; break;
case '*': tok = FTS5_STAR; break;
case '\0': tok = FTS5_EOF; break;
case '"': {
const char *z2;
tok = FTS5_STRING;
for(z2=&z[1]; 1; z2++){
if( z2[0]=='"' ){
z2++;
if( z2[0]!='"' ) break;
}
if( z2[0]=='\0' ){
sqlite3Fts5ParseError(pParse, "unterminated string");
return FTS5_EOF;
}
}
pToken->n = (z2 - z);
break;
}
default: {
const char *z2;
tok = FTS5_STRING;
for(z2=&z[1]; fts5ExprIstoken(*z2); z2++);
pToken->n = (z2 - z);
if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR;
if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT;
if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND;
break;
}
}
*pz = &pToken->p[pToken->n];
return tok;
}
static void *fts5ParseAlloc(size_t t){ return sqlite3_malloc((int)t); }
static void fts5ParseFree(void *p){ sqlite3_free(p); }
int sqlite3Fts5ExprNew(
Fts5Config *pConfig,
Fts5Index *pIdx,
const char *zExpr, /* Expression text */
Fts5Expr **ppNew,
char **pzErr
){
Fts5Parse sParse;
Fts5Token token;
const char *z = zExpr;
int t; /* Next token type */
void *pEngine;
*ppNew = 0;
*pzErr = 0;
memset(&sParse, 0, sizeof(sParse));
pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc);
if( pEngine==0 ) return SQLITE_NOMEM;
sParse.pConfig = pConfig;
do {
t = fts5ExprGetToken(&sParse, &z, &token);
sqlite3Fts5Parser(pEngine, t, token, &sParse);
}while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
assert( sParse.pExpr==0 || (sParse.rc==SQLITE_OK && sParse.zErr==0) );
*ppNew = sParse.pExpr;
*pzErr = sParse.zErr;
return sParse.rc;
}
/*
** Free the object passed as the only argument.
*/
void sqlite3Fts5ExprFree(Fts5Expr *p){
if( p ){
sqlite3Fts5ExprFree(p->pLeft);
sqlite3Fts5ExprFree(p->pRight);
sqlite3Fts5ParseNearsetFree(p->pNear);
sqlite3_free(p);
}
}
/*
** Argument pIn points to a buffer of nIn bytes. This function allocates
** and returns a new buffer populated with a copy of (pIn/nIn) with a
** nul-terminator byte appended to it.
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
*/
static char *fts5Strdup(const char *pIn, int nIn){
char *zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
zRet[nIn] = '\0';
}
return zRet;
}
static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
*pz = sqlite3_mprintf("%.*s", pToken->n, pToken->p);
if( *pz==0 ) return SQLITE_NOMEM;
return SQLITE_OK;
}
/*
** Free the phrase object passed as the only argument.
*/
static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
if( pPhrase ){
int i;
for(i=0; i<pPhrase->nTerm; i++){
sqlite3_free(pPhrase->aTerm[i].zTerm);
}
sqlite3_free(pPhrase);
}
}
/*
** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
** appended to it and the results returned.
**
** If an OOM error occurs, both the pNear and pPhrase objects are freed and
** NULL returned.
*/
Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5Parse *pParse, /* Parse context */
Fts5ExprNearset *pNear, /* Existing nearset, or NULL */
Fts5ExprPhrase *pPhrase /* Recently parsed phrase */
){
const int SZALLOC = 8;
Fts5ExprNearset *pRet = 0;
if( pParse->rc==SQLITE_OK ){
if( pNear==0 ){
int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
pRet = sqlite3_malloc(nByte);
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
pRet->iCol = -1;
}
}else if( (pNear->nPhrase % SZALLOC)==0 ){
int nNew = pRet->nPhrase + SZALLOC;
int nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*);
pRet = (Fts5ExprNearset*)sqlite3_realloc(pNear, nByte);
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}
}else{
pRet = pNear;
}
}
if( pRet==0 ){
assert( pParse->rc!=SQLITE_OK );
sqlite3Fts5ParseNearsetFree(pNear);
sqlite3Fts5ParsePhraseFree(pPhrase);
}else{
pRet->apPhrase[pRet->nPhrase++] = pPhrase;
}
return pRet;
}
typedef struct TokenCtx TokenCtx;
struct TokenCtx {
Fts5ExprPhrase *pPhrase;
};
/*
** Callback for tokenizing terms used by ParseTerm().
*/
static int fts5ParseTokenize(
void *pContext, /* Pointer to Fts5InsertCtx object */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd, /* End offset of token */
int iPos /* Position offset of token */
){
const int SZALLOC = 8;
TokenCtx *pCtx = (TokenCtx*)pContext;
Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
Fts5ExprTerm *pTerm;
if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
Fts5ExprPhrase *pNew;
int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase,
sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
);
if( pNew==0 ) return SQLITE_NOMEM;
pCtx->pPhrase = pPhrase = pNew;
pNew->nTerm = nNew - SZALLOC;
}
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
pTerm->bPrefix = 0;
pTerm->zTerm = fts5Strdup(pToken, nToken);
return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
}
/*
** Free the phrase object passed as the only argument.
*/
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){
fts5ExprPhraseFree(pPhrase);
}
/*
** Free the phrase object passed as the second argument.
*/
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){
if( pNear ){
int i;
for(i=0; i<pNear->nPhrase; i++){
fts5ExprPhraseFree(pNear->apPhrase[i]);
}
sqlite3_free(pNear);
}
}
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5Expr *p){
assert( pParse->pExpr==0 );
pParse->pExpr = p;
}
/*
** This function is called by the parser to process a string token. The
** string may or may not be quoted. In any case it is tokenized and a
** phrase object consisting of all tokens returned.
*/
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse, /* Parse context */
Fts5ExprPhrase *pPhrase, /* Phrase to append to */
Fts5Token *pToken, /* String to tokenize */
int bPrefix /* True if there is a trailing "*" */
){
Fts5Config *pConfig = pParse->pConfig;
TokenCtx sCtx; /* Context object passed to callback */
int rc; /* Tokenize return code */
char *z = 0;
pParse->rc = fts5ParseStringFromToken(pToken, &z);
if( z==0 ) return 0;
sqlite3Fts5Dequote(z);
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pPhrase;
rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
if( rc ){
pParse->rc = rc;
fts5ExprPhraseFree(sCtx.pPhrase);
sCtx.pPhrase = 0;
}else if( sCtx.pPhrase->nTerm>0 ){
sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
}
sqlite3_free(z);
return sCtx.pPhrase;
}
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){
if( pParse->rc==SQLITE_OK ){
if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){
sqlite3Fts5ParseError(
pParse, "syntax error near \"%.*s\"", pTok->n, pTok->p
);
}
}
}
void sqlite3Fts5ParseSetDistance(
Fts5Parse *pParse,
Fts5ExprNearset *pNear,
Fts5Token *p
){
int nNear = 0;
int i;
if( p->n ){
for(i=0; i<p->n; i++){
char c = (char)p->p[i];
if( c<'0' || c>'9' ){
sqlite3Fts5ParseError(
pParse, "expected integer, got \"%.*s\"", p->n, p->p
);
return;
}
nNear = nNear * 10 + (p->p[i] - '0');
}
}else{
nNear = FTS5_DEFAULT_NEARDIST;
}
pNear->nNear = nNear;
}
void sqlite3Fts5ParseSetColumn(
Fts5Parse *pParse,
Fts5ExprNearset *pNear,
Fts5Token *p
){
char *z = 0;
int rc = fts5ParseStringFromToken(p, &z);
if( rc==SQLITE_OK ){
Fts5Config *pConfig = pParse->pConfig;
int i;
for(i=0; i<pConfig->nCol; i++){
if( 0==sqlite3_stricmp(pConfig->azCol[i], z) ){
pNear->iCol = i;
break;
}
}
if( i==pConfig->nCol ){
sqlite3Fts5ParseError(pParse, "no such column: %s", z);
}
sqlite3_free(z);
}else{
pParse->rc = rc;
}
}
/*
** Allocate and return a new expression object. If anything goes wrong (i.e.
** OOM error), leave an error code in pParse and return NULL.
*/
Fts5Expr *sqlite3Fts5ParseExpr(
Fts5Parse *pParse, /* Parse context */
int eType, /* FTS5_STRING, AND, OR or NOT */
Fts5Expr *pLeft, /* Left hand child expression */
Fts5Expr *pRight, /* Right hand child expression */
Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */
){
Fts5Expr *pRet = 0;
if( pParse->rc==SQLITE_OK ){
assert( (eType!=FTS5_STRING && pLeft && pRight && !pNear)
|| (eType==FTS5_STRING && !pLeft && !pRight && pNear)
);
pRet = (Fts5Expr*)sqlite3_malloc(sizeof(Fts5Expr));
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, sizeof(*pRet));
pRet->eType = eType;
pRet->pLeft = pLeft;
pRet->pRight = pRight;
pRet->pNear = pNear;
}
}
if( pRet==0 ){
assert( pParse->rc!=SQLITE_OK );
sqlite3Fts5ExprFree(pLeft);
sqlite3Fts5ExprFree(pRight);
sqlite3Fts5ParseNearsetFree(pNear);
}
return pRet;
}
static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
char *zQuoted = sqlite3_malloc(strlen(pTerm->zTerm) * 2 + 3 + 2);
if( zQuoted ){
int i = 0;
char *zIn = pTerm->zTerm;
zQuoted[i++] = '"';
while( *zIn ){
if( *zIn=='"' ) zQuoted[i++] = '"';
zQuoted[i++] = *zIn++;
}
zQuoted[i++] = '"';
if( pTerm->bPrefix ){
zQuoted[i++] = ' ';
zQuoted[i++] = '*';
}
zQuoted[i++] = '\0';
}
return zQuoted;
}
static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){
char *zNew;
va_list ap;
va_start(ap, zFmt);
zNew = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zApp ){
char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew);
sqlite3_free(zNew);
zNew = zNew2;
}
sqlite3_free(zApp);
return zNew;
}
static char *fts5ExprPrint(Fts5Config *pConfig, Fts5Expr *pExpr){
char *zRet = 0;
if( pExpr->eType==FTS5_STRING ){
Fts5ExprNearset *pNear = pExpr->pNear;
int i;
int iTerm;
if( pNear->iCol>=0 ){
zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[pNear->iCol]);
if( zRet==0 ) return 0;
}
if( pNear->nPhrase>1 ){
zRet = fts5PrintfAppend(zRet, "NEAR(");
if( zRet==0 ) return 0;
}
for(i=0; i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
if( i!=0 ){
zRet = fts5PrintfAppend(zRet, " ");
if( zRet==0 ) return 0;
}
for(iTerm=0; iTerm<pPhrase->nTerm; iTerm++){
char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]);
if( zTerm ){
zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm);
sqlite3_free(zTerm);
}
if( zTerm==0 || zRet==0 ){
sqlite3_free(zRet);
return 0;
}
}
}
if( pNear->nPhrase>1 ){
zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear);
if( zRet==0 ) return 0;
}
}else{
char *zOp = 0;
char *z1 = 0;
char *z2 = 0;
switch( pExpr->eType ){
case FTS5_AND: zOp = "AND"; break;
case FTS5_NOT: zOp = "NOT"; break;
case FTS5_OR: zOp = "OR"; break;
default: assert( 0 );
}
z1 = fts5ExprPrint(pConfig, pExpr->pLeft);
z2 = fts5ExprPrint(pConfig, pExpr->pRight);
if( z1 && z2 ){
int b1 = pExpr->pLeft->eType!=FTS5_STRING;
int b2 = pExpr->pRight->eType!=FTS5_STRING;
zRet = sqlite3_mprintf("%s%s%s %s %s%s%s",
b1 ? "(" : "", z1, b1 ? ")" : "",
zOp,
b2 ? "(" : "", z2, b2 ? ")" : ""
);
}
sqlite3_free(z1);
sqlite3_free(z2);
}
return zRet;
}
/*
** The implementation of user-defined scalar function fts5_expr().
*/
static void fts5ExprFunction(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
sqlite3 *db = sqlite3_context_db_handle(pCtx);
const char *zExpr = 0;
char *zErr = 0;
Fts5Expr *pExpr = 0;
int rc;
int i;
const char **azConfig; /* Array of arguments for Fts5Config */
int nConfig; /* Size of azConfig[] */
Fts5Config *pConfig = 0;
nConfig = nArg + 2;
azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig);
if( azConfig==0 ){
sqlite3_result_error_nomem(pCtx);
return;
}
azConfig[0] = 0;
azConfig[1] = "main";
azConfig[2] = "tbl";
for(i=1; i<nArg; i++){
azConfig[i+2] = (const char*)sqlite3_value_text(apVal[i]);
}
zExpr = (const char*)sqlite3_value_text(apVal[0]);
rc = sqlite3Fts5ConfigParse(db, nConfig, azConfig, &pConfig, &zErr);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprNew(pConfig, 0, zExpr, &pExpr, &zErr);
}
if( rc==SQLITE_OK ){
char *zText = fts5ExprPrint(pConfig, pExpr);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
sqlite3_free(zText);
}
}
if( rc!=SQLITE_OK ){
if( zErr ){
sqlite3_result_error(pCtx, zErr, -1);
sqlite3_free(zErr);
}else{
sqlite3_result_error_code(pCtx, rc);
}
}
sqlite3_free(azConfig);
sqlite3Fts5ConfigFree(pConfig);
sqlite3Fts5ExprFree(pExpr);
}
/*
** This is called during initialization to register the fts5_expr() scalar
** UDF with the SQLite handle passed as the only argument.
*/
int sqlite3Fts5ExprInit(sqlite3 *db){
int rc = sqlite3_create_function(
db, "fts5_expr", -1, SQLITE_UTF8, 0, fts5ExprFunction, 0, 0
);
return rc;
}

2939
ext/fts5/fts5_index.c Normal file
View File

File diff suppressed because it is too large Load Diff

411
ext/fts5/fts5_storage.c Normal file
View File

@ -0,0 +1,411 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#include "fts5Int.h"
struct Fts5Storage {
Fts5Config *pConfig;
Fts5Index *pIndex;
sqlite3_stmt *aStmt[7];
};
#define FTS5_STMT_INSERT_CONTENT 0
#define FTS5_STMT_REPLACE_CONTENT 1
#define FTS5_STMT_DELETE_CONTENT 2
#define FTS5_STMT_INSERT_DOCSIZE 3
#define FTS5_STMT_DELETE_DOCSIZE 4
#define FTS5_STMT_SCAN_CONTENT 5
#define FTS5_STMT_SEEK_CONTENT 6
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int fts5StorageGetStmt(
Fts5Storage *p, /* Storage handle */
int eStmt, /* FTS5_STMT_XXX constant */
sqlite3_stmt **ppStmt /* OUT: Prepared statement handle */
){
int rc = SQLITE_OK;
assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
if( p->aStmt[eStmt]==0 ){
const char *azStmt[] = {
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
"DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
"INSERT INTO %Q.'%q_docsize' VALUES(?,?)", /* INSERT_DOCSIZE */
"DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
"SELECT * FROM %Q.'%q_content'", /* SCAN_CONTENT */
"SELECT * FROM %Q.'%q_content' WHERE rowid=?", /* SEEK_CONTENT */
};
Fts5Config *pConfig = p->pConfig;
char *zSql = 0;
if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
int nCol = pConfig->nCol + 1;
char *zBind;
int i;
zBind = sqlite3_malloc(1 + nCol*2);
if( zBind ){
for(i=0; i<nCol; i++){
zBind[i*2] = '?';
zBind[i*2 + 1] = ',';
}
zBind[i*2-1] = '\0';
zSql = sqlite3_mprintf(azStmt[eStmt],pConfig->zDb,pConfig->zName,zBind);
sqlite3_free(zBind);
}
}else{
zSql = sqlite3_mprintf(azStmt[eStmt], pConfig->zDb, pConfig->zName);
}
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->aStmt[eStmt], 0);
sqlite3_free(zSql);
}
}
*ppStmt = p->aStmt[eStmt];
return rc;
}
/*
** Drop the shadow table with the postfix zPost (e.g. "content"). Return
** SQLITE_OK if successful or an SQLite error code otherwise.
*/
int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){
int rc;
char *zSql = sqlite3_mprintf("DROP TABLE IF EXISTS %Q.'%q_%q'",
pConfig->zDb, pConfig->zName, zPost
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_exec(pConfig->db, zSql, 0, 0, 0);
sqlite3_free(zSql);
}
return rc;
}
/*
** Create the shadow table named zPost, with definition zDefn. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
*/
int sqlite3Fts5CreateTable(
Fts5Config *pConfig, /* FTS5 configuration */
const char *zPost, /* Shadow table to create (e.g. "content") */
const char *zDefn, /* Columns etc. for shadow table */
char **pzErr /* OUT: Error message */
){
int rc;
char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)",
pConfig->zDb, pConfig->zName, zPost, zDefn
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
char *zErr = 0;
assert( *pzErr==0 );
rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
if( zErr ){
*pzErr = sqlite3_mprintf(
"fts5: error creating shadow table %q_%s: %s",
pConfig->zName, zPost, zErr
);
sqlite3_free(zErr);
}
sqlite3_free(zSql);
}
return rc;
}
/*
** Open a new Fts5Index handle. If the bCreate argument is true, create
** and initialize the underlying tables
**
** If successful, set *pp to point to the new object and return SQLITE_OK.
** Otherwise, set *pp to NULL and return an SQLite error code.
*/
int sqlite3Fts5StorageOpen(
Fts5Config *pConfig,
Fts5Index *pIndex,
int bCreate,
Fts5Storage **pp,
char **pzErr /* OUT: Error message */
){
int rc;
Fts5Storage *p; /* New object */
*pp = p = (Fts5Storage*)sqlite3_malloc(sizeof(Fts5Storage));
if( !p ) return SQLITE_NOMEM;
memset(p, 0, sizeof(Fts5Storage));
p->pConfig = pConfig;
p->pIndex = pIndex;
if( bCreate ){
int i;
char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
for(i=0; i<pConfig->nCol; i++){
iOff += sprintf(&zDefn[iOff], ", c%d", i);
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, pzErr);
}
sqlite3_free(zDefn);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5CreateTable(
pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", pzErr
);
}
}
if( rc ){
sqlite3Fts5StorageClose(p, 0);
*pp = 0;
}
return rc;
}
/*
** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
*/
int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy){
int rc = SQLITE_OK;
int i;
/* Finalize all SQL statements */
for(i=0; i<ArraySize(p->aStmt); i++){
sqlite3_finalize(p->aStmt[i]);
}
/* If required, remove the shadow tables from the database */
if( bDestroy ){
rc = sqlite3Fts5DropTable(p->pConfig, "content");
if( rc==SQLITE_OK ) sqlite3Fts5DropTable(p->pConfig, "docsize");
}
sqlite3_free(p);
return rc;
}
/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
assert( !"do this" );
return SQLITE_OK;
}
typedef struct Fts5InsertCtx Fts5InsertCtx;
struct Fts5InsertCtx {
Fts5Storage *pStorage;
int iCol;
};
/*
** Tokenization callback used when inserting tokens into the FTS index.
*/
static int fts5StorageInsertCallback(
void *pContext, /* Pointer to Fts5InsertCtx object */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd, /* End offset of token */
int iPos /* Position offset of token */
){
Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
Fts5Index *pIdx = pCtx->pStorage->pIndex;
sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
return SQLITE_OK;
}
/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
*/
static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
Fts5Config *pConfig = p->pConfig;
sqlite3_stmt *pSeek; /* SELECT to read row iDel from %_data */
int rc; /* Return code */
rc = fts5StorageGetStmt(p, FTS5_STMT_SEEK_CONTENT, &pSeek);
if( rc==SQLITE_OK ){
int rc2;
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)==SQLITE_ROW ){
int iCol;
Fts5InsertCtx ctx;
ctx.pStorage = p;
ctx.iCol = -1;
sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
for(iCol=1; iCol<=pConfig->nCol; iCol++){
rc = sqlite3Fts5Tokenize(pConfig,
(const char*)sqlite3_column_text(pSeek, iCol),
sqlite3_column_bytes(pSeek, iCol),
(void*)&ctx,
fts5StorageInsertCallback
);
}
}
rc2 = sqlite3_reset(pSeek);
if( rc==SQLITE_OK ) rc = rc2;
}
return rc;
}
/*
** Insert a new row into the FTS table.
*/
int sqlite3Fts5StorageInsert(
Fts5Storage *p, /* Storage module to write to */
sqlite3_value **apVal, /* Array of values passed to xUpdate() */
int eConflict, /* on conflict clause */
i64 *piRowid /* OUT: rowid of new record */
){
Fts5Config *pConfig = p->pConfig;
int rc = SQLITE_OK; /* Return code */
sqlite3_stmt *pInsert; /* Statement used to write %_content table */
int eStmt; /* Type of statement used on %_content */
int i; /* Counter variable */
Fts5InsertCtx ctx; /* Tokenization callback context object */
/* Insert the new row into the %_content table. */
if( eConflict==SQLITE_REPLACE ){
eStmt = FTS5_STMT_REPLACE_CONTENT;
if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
}
}else{
eStmt = FTS5_STMT_INSERT_CONTENT;
}
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, eStmt, &pInsert);
}
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
rc = sqlite3_bind_value(pInsert, i, apVal[i]);
}
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
rc = sqlite3_reset(pInsert);
}
*piRowid = sqlite3_last_insert_rowid(pConfig->db);
/* Add new entries to the FTS index */
sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
ctx.pStorage = p;
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
rc = sqlite3Fts5Tokenize(pConfig,
(const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
sqlite3_value_bytes(apVal[ctx.iCol+2]),
(void*)&ctx,
fts5StorageInsertCallback
);
}
return rc;
}
/*
** Context object used by sqlite3Fts5StorageIntegrity().
*/
typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
struct Fts5IntegrityCtx {
i64 iRowid;
int iCol;
u64 cksum;
Fts5Config *pConfig;
};
/*
** Tokenization callback used by integrity check.
*/
static int fts5StorageIntegrityCallback(
void *pContext, /* Pointer to Fts5InsertCtx object */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd, /* End offset of token */
int iPos /* Position offset of token */
){
Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
pCtx->cksum ^= sqlite3Fts5IndexCksum(
pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken
);
return SQLITE_OK;
}
/*
** Check that the contents of the FTS index match that of the %_content
** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
** some other SQLite error code if an error occurs while attempting to
** determine this.
*/
int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
Fts5Config *pConfig = p->pConfig;
int rc; /* Return code */
Fts5IntegrityCtx ctx;
sqlite3_stmt *pScan;
memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
ctx.pConfig = p->pConfig;
/* Generate the expected index checksum based on the contents of the
** %_content table. This block stores the checksum in ctx.cksum. */
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_CONTENT, &pScan);
if( rc==SQLITE_OK ){
int rc2;
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
ctx.iCol = i;
rc = sqlite3Fts5Tokenize(
pConfig,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
}
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
/* Pass the expected checksum down to the FTS index module. It will
** verify, amongst other things, that it matches the checksum generated by
** inspecting the index itself. */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
}
return rc;
}

33
main.mk
View File

@ -47,6 +47,7 @@
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
TCCX += -I$(TOP)/ext/async
TCCX += -I$(TOP)/ext/fts5
# Object files for the SQLite library.
#
@ -71,6 +72,13 @@ LIBOBJ+= vdbe.o parse.o \
vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
vdbetrace.o wal.o walker.o where.o utf.o vtab.o
LIBOBJ += fts5.o
LIBOBJ += fts5_config.o
LIBOBJ += fts5_expr.o
LIBOBJ += fts5_index.o
LIBOBJ += fts5_storage.o
LIBOBJ += fts5parse.o
# All of the source code files.
@ -375,6 +383,8 @@ EXTHDR += \
$(TOP)/ext/rtree/rtree.h
EXTHDR += \
$(TOP)/ext/icu/sqliteicu.h
EXTHDR += \
$(TOP)/ext/fts5/fts5Int.h
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
@ -553,10 +563,33 @@ fts3_unicode2.o: $(TOP)/ext/fts3/fts3_unicode2.c $(HDR) $(EXTHDR)
fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
fts5.o: $(TOP)/ext/fts5/fts5.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5.c
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
# FTS5 things
#
fts5_config.o: $(TOP)/ext/fts5/fts5_config.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_config.c
fts5_expr.o: $(TOP)/ext/fts5/fts5_expr.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_expr.c
fts5_index.o: $(TOP)/ext/fts5/fts5_index.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_index.c
fts5_storage.o: $(TOP)/ext/fts5/fts5_storage.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
./lemon $(OPTS) fts5parse.y
# Rules for building test programs and for running tests
#
tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a

27
manifest
View File

@ -1,5 +1,5 @@
C Fix\sa\sproblem\swith\sSQLITE_OMIT_WSD\sbuilds.
D 2014-06-23T10:18:50.447
C Add\ssome\scode\sfor\san\sexperimental\sfts5\smodule.\sDoes\snot\swork\syet.
D 2014-06-23T11:33:22.754
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -79,7 +79,7 @@ F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 20bc65862cfcea0a39bb64a819f8fe92a8e144c1
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3.h 62a77d880cf06a2865052726f8325c8fabcecad7
F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918
F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365
F ext/fts3/fts3_expr.c 351395fad6fcb16ecfc61db0861008a70101330c
@ -103,6 +103,12 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
F ext/fts5/fts5.c 2cb2cc3c1acefa36d9e8ce8e68bceaac8515059a
F ext/fts5/fts5Int.h cc41cf776a3e612aa3a461e96463647fd3957bed
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
F ext/fts5/fts5_expr.c bdfb98dab7729cf967022d7a4a815828bbad8c23
F ext/fts5/fts5_index.c 0548e8925a0664cfa00b2477ebe9afa18bc7848f
F ext/fts5/fts5_storage.c aa1ff4b6b283303ffd8c5dc57a45ebe55e62a7b2
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@ -146,7 +152,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F main.mk 7b1d0be0840f213405c977c87917241158126a33
F main.mk 2bb1ec703ac4f27743961764b59cfb5f91d72bfe
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@ -189,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12
F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303
F src/main.c 7c2c3cafdd6313c8f9319ebec1565782e624372e
F src/main.c e777879ad7c431f5b3b5d49c8419727b61d7c1be
F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b
@ -585,6 +591,8 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
F test/fts5aa.test bbea71fed733b1d433bf83dbc8d86077936d1efc
F test/fts5ea.test 814287a2cb25ac3e59abbe4ccbcabf6bda821868
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@ -1179,7 +1187,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 612b6d1b1f74eaf618520b90811eca10f978fc71
R 283ec0802d51fd4e82222d529f6a8475
P 07dda49c1bf8997a18c3368acb81b6d863ea38d6
R 66e5d0ccaa728e4d98b92edeb331ffb3
T *branch * fts5
T *sym-fts5 *
T -sym-trunk *
U dan
Z de359222916ca6f6bd684ca986937509
Z e3b7f827041011d2f1d78b39cdee11d7

2
manifest.uuid
View File

@ -1 +1 @@
07dda49c1bf8997a18c3368acb81b6d863ea38d6
1e0648dcf283d4f1f6159db4d2433b6cc635992e

1
src/main.c
View File

@ -2609,6 +2609,7 @@ static int openDatabase(
#ifdef SQLITE_ENABLE_FTS3
if( !db->mallocFailed && rc==SQLITE_OK ){
rc = sqlite3Fts3Init(db);
if( rc==SQLITE_OK ) rc = sqlite3Fts5Init(db);
}
#endif

248
test/fts5aa.test Normal file
View File

@ -0,0 +1,248 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5aa
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
SELECT name, sql FROM sqlite_master;
} {
t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
}
do_execsql_test 1.1 {
DROP TABLE t1;
SELECT name, sql FROM sqlite_master;
} {
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
}
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
}
do_execsql_test 2.2 {
SELECT fts5_decode(id, block) FROM t1_data WHERE id==10
} {
{{structure idx=0} {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}}
}
do_execsql_test 2.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
} {
do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
INSERT INTO t1(t1) VALUES('pgsz=32');
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
} {
do_execsql_test 4.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 4.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
INSERT INTO t1(t1) VALUES('pgsz=32');
}
foreach {i x y} {
1 {dd abc abc abc abcde} {aaa dd ddd ddd aab}
2 {dd aab d aaa b} {abcde c aaa aaa aaa}
3 {abcde dd b b dd} {abc abc d abc ddddd}
4 {aaa abcde dddd dddd abcde} {abc b b abcde abc}
5 {aab dddd d dddd c} {ddd abcde dddd abcde c}
6 {ddd dd b aab abcde} {d ddddd dddd c abc}
7 {d ddddd ddd c abcde} {c aab d abcde ddd}
8 {abcde aaa aab c c} {ddd c dddd b aaa}
9 {abcde aab ddddd c aab} {dddd dddd b c dd}
10 {ddd abcde dddd dd c} {dddd c c d abcde}
} {
do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
INSERT INTO t1(t1) VALUES('pgsz=32');
}
do_execsql_test 6.1 {
INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
}
do_execsql_test 6.2 {
INSERT INTO t1(t1) VALUES('integrity-check')
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
INSERT INTO t1(t1) VALUES('pgsz=32');
}
proc doc {} {
set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
set ret [list]
for {set j 0} {$j < 20} {incr j} {
lappend ret [lindex $v [expr int(rand()*[llength $v])]]
}
return $ret
}
proc dump_structure {} {
db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
foreach lvl [lrange $t 1 end] {
set seg [string repeat . [expr [llength $lvl]-2]]
puts "[lrange $lvl 0 1] $seg"
}
}
}
for {set i 1} {$i <= 10} {incr i} {
do_test 7.$i {
for {set j 0} {$j < 100} {incr j} {
set x [doc]
set y [doc]
set z [doc]
set rowid [expr int(rand() * 100)]
execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
} {}
if {[set_test_counter errors]} exit
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
INSERT INTO t1(t1) VALUES('pgsz=32');
}
do_execsql_test 8.1 {
INSERT INTO t1 VALUES('the quick brown fox');
INSERT INTO t1(t1) VALUES('integrity-check');
}
#finish_test
#-------------------------------------------------------------------------
#
reset_db
expr srand(0)
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
INSERT INTO t1(t1) VALUES('pgsz=32');
}
proc doc {} {
set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
set ret [list]
for {set j 0} {$j < 20} {incr j} {
lappend ret [lindex $v [expr int(rand()*[llength $v])]]
}
return $ret
}
proc dump_structure {} {
db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
foreach lvl [lrange $t 1 end] {
set seg [string repeat . [expr [llength $lvl]-2]]
puts "[lrange $lvl 0 1] $seg"
}
}
}
for {set i 1} {$i <= 10} {incr i} {
do_test 9.$i {
for {set j 0} {$j < 100} {incr j} {
set x [doc]
set y [doc]
set z [doc]
set rowid [expr int(rand() * 100)]
execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
} {}
if {[set_test_counter errors]} break
}
finish_test

84
test/fts5ea.test Normal file
View File

@ -0,0 +1,84 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5ea
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
proc do_syntax_error_test {tn expr err} {
set ::se_expr $expr
do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]
}
proc do_syntax_test {tn expr res} {
set ::se_expr $expr
do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
}
foreach {tn expr res} {
1 {abc} {"abc"}
2 {abc def} {"abc" AND "def"}
3 {abc*} {"abc" *}
4 {"abc def ghi" *} {"abc" + "def" + "ghi" *}
5 {one AND two} {"one" AND "two"}
6 {one+two} {"one" + "two"}
7 {one AND two OR three} {("one" AND "two") OR "three"}
8 {one OR two AND three} {"one" OR ("two" AND "three")}
9 {NEAR(one two)} {NEAR("one" "two", 10)}
10 {NEAR("one three"* two, 5)} {NEAR("one" + "three" * "two", 5)}
} {
do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}
foreach {tn expr res} {
1 {c1:abc}
{c1 : "abc"}
2 {c2 : NEAR(one two) c1:"hello world"}
{c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"}
} {
do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res]
}
breakpoint
foreach {tn expr err} {
1 {AND} {syntax error near "AND"}
2 {abc def AND} {syntax error near ""}
3 {abc OR AND} {syntax error near "AND"}
4 {(a OR b) abc} {syntax error near "abc"}
5 {NEaR (a b)} {syntax error near "NEaR"}
6 {(a OR b) NOT c)} {syntax error near ")"}
7 {nosuch: a nosuch2: b} {no such column: nosuch}
8 {addr: a nosuch2: b} {no such column: nosuch2}
} {
do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
}
# do_syntax_error_test 1.0 {NOT} {syntax error near "NOT"}
# do_catchsql_test 1.1 {
# SELECT fts5_expr('a OR b NOT c')
#} {0 {"a" OR "b" NOT "c"}}
#do_execsql_test 1.0 { SELECT fts5_expr('a') } {{"a"}}
finish_test