mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for external content tables to fts5.

Browse Source 
FossilOrigin-Name: 17ef5b59f789e9fa35c4f053246d819987fd06f8
This commit is contained in:
dan 2015-01-03 20:44:58 +00:00
parent ade921c3ad
commit 0fbc269fef
7 changed files with 283 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ext/fts5/fts5.c
View File

@ -1056,6 +1056,20 @@ static int fts5SpecialInsert(
return rc;
}
static int fts5SpecialDelete(
Fts5Table *pTab,
sqlite3_value **apVal,
sqlite3_int64 *piRowid
){
int rc = SQLITE_OK;
int eType1 = sqlite3_value_type(apVal[1]);
if( eType1==SQLITE_INTEGER ){
sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
}
return rc;
}
/*
** This function is the implementation of the xUpdate callback used by
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
@ -1086,17 +1100,25 @@ static int fts5UpdateMethod(
*/
assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
return fts5SpecialInsert(pTab,
apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
);
if( nArg>1 ){
sqlite3_value *pCmd = sqlite3_value_type(apVal[2 + pConfig->nCol]);
if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
const char *z = sqlite3_value_text(pCmd);
if( pConfig->bExternalContent && sqlite3_stricmp("delete", z) ){
return fts5SpecialDelete(pTab, apVal, pRowid);
}else{
return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);
}
}
}
eType0 = sqlite3_value_type(apVal[0]);
eConflict = sqlite3_vtab_on_conflict(pConfig->db);
assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
if( eType0==SQLITE_INTEGER ){
assert( pVtab->zErrMsg==0 );
if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
}

5
ext/fts5/fts5Int.h
View File

@ -76,6 +76,9 @@ struct Fts5Config {
char **azCol; /* Column names */
int nPrefix; /* Number of prefix indexes */
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
int bExternalContent; /* Content is external */
char *zContent; /* "content=" option value (or NULL) */
char *zContentRowid; /* "content_rowid=" option value (or NULL) */
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
@ -410,6 +413,8 @@ int sqlite3Fts5StorageRollback(Fts5Storage *p);
int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/

96
ext/fts5/fts5_config.c
View File

@ -192,6 +192,47 @@ static char *fts5TrimString(char *z){
return z;
}
/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(int *pRc, const char *z){
char *pRet = 0;
if( *pRc==SQLITE_OK ){
pRet = sqlite3_mprintf("%s", z);
if( pRet==0 ) *pRc = SQLITE_NOMEM;
}
return pRet;
}
/*
** Argument z points to a nul-terminated string containing an SQL identifier.
** This function returns a copy of the identifier enclosed in backtick
** quotes.
*/
static char *fts5EscapeName(int *pRc, const char *z){
char *pRet = 0;
if( *pRc==SQLITE_OK ){
int n = strlen(z);
pRet = (char*)sqlite3_malloc(2 * 2*n + 1);
if( pRet==0 ){
*pRc = SQLITE_NOMEM;
}else{
int i;
char *p = pRet;
for(i=0; i<n; i++){
if( z[i]=='`' ) *p++ = '`';
*p++ = z[i];
}
*p++ = '`';
*p++ = '\0';
}
}
return pRet;
}
/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
@ -291,25 +332,34 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
int rc = SQLITE_OK;
if( pConfig->zContent ){
*pzErr = sqlite3_mprintf("multiple content=... directives");
rc = SQLITE_ERROR;
}else{
pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
pConfig->bExternalContent = 1;
if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
}
return rc;
}
if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
int rc = SQLITE_OK;
if( pConfig->zContentRowid ){
*pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
rc = SQLITE_ERROR;
}else{
pConfig->zContentRowid = fts5EscapeName(&rc, zArg);
}
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
return SQLITE_ERROR;
}
/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(int *pRc, const char *z){
char *pRet = 0;
if( *pRc==SQLITE_OK ){
pRet = sqlite3_mprintf("%s", z);
if( pRet==0 ) *pRc = SQLITE_NOMEM;
}
return pRet;
}
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
@ -422,6 +472,20 @@ int sqlite3Fts5ConfigParse(
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
if( rc==SQLITE_OK && pRet->zContent==0 ){
pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
if( pRet->zContent==0 ){
rc = SQLITE_NOMEM;
}else{
sqlite3_free(pRet->zContentRowid);
pRet->zContentRowid = 0;
}
}
if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
pRet->zContentRowid = fts5Strdup(&rc, "rowid");
}
if( rc!=SQLITE_OK ){
sqlite3Fts5ConfigFree(pRet);
*ppOut = 0;
@ -447,6 +511,8 @@ void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
sqlite3_free(pConfig->aPrefix);
sqlite3_free(pConfig->zRank);
sqlite3_free(pConfig->zRankArgs);
sqlite3_free(pConfig->zContent);
sqlite3_free(pConfig->zContentRowid);
sqlite3_free(pConfig);
}
}

26
ext/fts5/fts5_expr.c
View File

@ -1041,19 +1041,24 @@ i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
*/
static char *fts5Strndup(const char *pIn, int nIn){
char *zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
zRet[nIn] = '\0';
static char *fts5Strndup(int *pRc, const char *pIn, int nIn){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
zRet[nIn] = '\0';
}else{
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
*pz = fts5Strndup(pToken->p, pToken->n);
if( *pz==0 ) return SQLITE_NOMEM;
return SQLITE_OK;
int rc = SQLITE_OK;
*pz = fts5Strndup(&rc, pToken->p, pToken->n);
return rc;
}
/*
@ -1139,6 +1144,7 @@ static int fts5ParseTokenize(
int iEnd, /* End offset of token */
int iPos /* Position offset of token */
){
int rc = SQLITE_OK;
const int SZALLOC = 8;
TokenCtx *pCtx = (TokenCtx*)pContext;
Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
@ -1159,9 +1165,9 @@ static int fts5ParseTokenize(
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
memset(pTerm, 0, sizeof(Fts5ExprTerm));
pTerm->zTerm = fts5Strndup(pToken, nToken);
pTerm->zTerm = fts5Strndup(&rc, pToken, nToken);
return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
return rc;
}

188
ext/fts5/fts5_storage.c
View File

@ -61,9 +61,9 @@ static int fts5StorageGetStmt(
assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
if( p->aStmt[eStmt]==0 ){
const char *azStmt[] = {
"SELECT * FROM %Q.'%q_content' ORDER BY id ASC", /* SCAN_ASC */
"SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */
"SELECT * FROM %Q.'%q_content' WHERE rowid=?", /* LOOKUP */
"SELECT * FROM %s ORDER BY id ASC", /* SCAN_ASC */
"SELECT * FROM %s ORDER BY id DESC", /* SCAN_DESC */
"SELECT * FROM %s WHERE %s=?", /* LOOKUP */
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
@ -75,32 +75,47 @@ static int fts5StorageGetStmt(
"REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
};
Fts5Config *pConfig = p->pConfig;
Fts5Config *pC = p->pConfig;
char *zSql = 0;
if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
int nCol = pConfig->nCol + 1;
char *zBind;
int i;
switch( eStmt ){
case FTS5_STMT_SCAN_ASC:
case FTS5_STMT_SCAN_DESC:
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent);
break;
zBind = sqlite3_malloc(1 + nCol*2);
if( zBind ){
for(i=0; i<nCol; i++){
zBind[i*2] = '?';
zBind[i*2 + 1] = ',';
case FTS5_STMT_LOOKUP:
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent, pC->zContentRowid);
break;
case FTS5_STMT_INSERT_CONTENT:
case FTS5_STMT_REPLACE_CONTENT: {
int nCol = pC->nCol + 1;
char *zBind;
int i;
zBind = sqlite3_malloc(1 + nCol*2);
if( zBind ){
for(i=0; i<nCol; i++){
zBind[i*2] = '?';
zBind[i*2 + 1] = ',';
}
zBind[i*2-1] = '\0';
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
sqlite3_free(zBind);
}
zBind[i*2-1] = '\0';
zSql = sqlite3_mprintf(azStmt[eStmt],pConfig->zDb,pConfig->zName,zBind);
sqlite3_free(zBind);
break;
}
}else{
zSql = sqlite3_mprintf(azStmt[eStmt], pConfig->zDb, pConfig->zName);
default:
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
break;
}
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->aStmt[eStmt], 0);
rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
sqlite3_free(zSql);
}
}
@ -190,18 +205,21 @@ int sqlite3Fts5StorageOpen(
p->pIndex = pIndex;
if( bCreate ){
int i;
char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
for(i=0; i<pConfig->nCol; i++){
iOff += sprintf(&zDefn[iOff], ", c%d", i);
if( pConfig->bExternalContent==0 ){
char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
for(i=0; i<pConfig->nCol; i++){
iOff += sprintf(&zDefn[iOff], ", c%d", i);
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
sqlite3_free(zDefn);
}
sqlite3_free(zDefn);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5CreateTable(
pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
@ -432,6 +450,78 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
return rc;
}
int sqlite3Fts5StorageSpecialDelete(
Fts5Storage *p,
i64 iDel,
sqlite3_value **apVal
){
Fts5Config *pConfig = p->pConfig;
int rc;
sqlite3_stmt *pDel;
assert( p->pConfig->bExternalContent );
rc = fts5StorageLoadTotals(p, 1);
/* Delete the index records */
if( rc==SQLITE_OK ){
int iCol;
Fts5InsertCtx ctx;
ctx.pStorage = p;
ctx.iCol = -1;
rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
rc = sqlite3Fts5Tokenize(pConfig,
(const char*)sqlite3_value_text(apVal[iCol]),
sqlite3_value_bytes(apVal[iCol]),
(void*)&ctx,
fts5StorageInsertCallback
);
p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
}
p->nTotalRow--;
}
/* Delete the %_docsize record */
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel);
}
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pDel, 1, iDel);
sqlite3_step(pDel);
rc = sqlite3_reset(pDel);
}
/* Write the averages record */
if( rc==SQLITE_OK ){
rc = fts5StorageSaveTotals(p);
}
return rc;
}
/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
** by inserting a dummy row into the %_docsize table. The dummy will be
** overwritten later.
*/
static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
sqlite3_stmt *pReplace = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
if( rc==SQLITE_OK ){
sqlite3_bind_null(pReplace, 1);
sqlite3_bind_null(pReplace, 2);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
}
if( rc==SQLITE_OK ){
*piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
}
return rc;
}
/*
** Insert a new row into the FTS table.
*/
@ -453,27 +543,35 @@ int sqlite3Fts5StorageInsert(
rc = fts5StorageLoadTotals(p, 1);
/* Insert the new row into the %_content table. */
if( rc==SQLITE_OK ){
if( eConflict==SQLITE_REPLACE ){
eStmt = FTS5_STMT_REPLACE_CONTENT;
if( rc==SQLITE_OK && pConfig->bExternalContent==0 ){
if( pConfig->bExternalContent ){
if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
*piRowid = sqlite3_value_int64(apVal[1]);
}else{
rc = fts5StorageNewRowid(p, piRowid);
}
}else{
eStmt = FTS5_STMT_INSERT_CONTENT;
if( eConflict==SQLITE_REPLACE ){
eStmt = FTS5_STMT_REPLACE_CONTENT;
if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
}
}else{
eStmt = FTS5_STMT_INSERT_CONTENT;
}
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, eStmt, &pInsert);
}
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
rc = sqlite3_bind_value(pInsert, i, apVal[i]);
}
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
rc = sqlite3_reset(pInsert);
}
*piRowid = sqlite3_last_insert_rowid(pConfig->db);
}
}
if( rc==SQLITE_OK ){
rc = fts5StorageGetStmt(p, eStmt, &pInsert);
}
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
rc = sqlite3_bind_value(pInsert, i, apVal[i]);
}
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
rc = sqlite3_reset(pInsert);
}
*piRowid = sqlite3_last_insert_rowid(pConfig->db);
/* Add new entries to the FTS index */
if( rc==SQLITE_OK ){

20
manifest
View File

@ -1,5 +1,5 @@
C Allow\sthe\srank\scolumn\sto\sbe\sremapped\son\sa\sper-query\sbasis\sby\sincluding\sa\sterm\ssimilar\sto\s"rank\smatch\s'bm25(10,2)'"\sin\sa\swhere\sclause.
D 2015-01-02T14:55:22.175
C Add\ssupport\sfor\sexternal\scontent\stables\sto\sfts5.
D 2015-01-03T20:44:58.134
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -104,16 +104,16 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e
F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
F ext/fts5/fts5.c a80283dca24506f1c748fffbba8d87ae4d348b50
F ext/fts5/fts5.c 16177d7f81af1852cf7f477b5ae119215ad6044a
F ext/fts5/fts5.h 4f9d2c477c0ee1907164642471329a82cb6b203b
F ext/fts5/fts5Int.h b5d7970b851d2b4f1745cd2d5c95216c9847aef2
F ext/fts5/fts5Int.h 8062dc2363c863dc8a5b2e5651cb8c966bd6c4cb
F ext/fts5/fts5_aux.c 445e54031ff94174673f4f5aac6c064df20a2a6b
F ext/fts5/fts5_buffer.c 1bc5c762bb2e9b4a40b2e8a820a31b809e72eec1
F ext/fts5/fts5_config.c 74a860e10c5583831f04d0088c4a49a3c6eca43d
F ext/fts5/fts5_expr.c 27d3d2deebae277c34ae2bb3d501dd879c442ba5
F ext/fts5/fts5_config.c 16d647c7bfe50d4e823267188e12e2d001d655e0
F ext/fts5/fts5_expr.c 317093f00a2ccdaaee0a5290f9f228c600189c41
F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
F ext/fts5/fts5_index.c 4a8e8535b4303400ddb5f6fb08152da0d88ebf6f
F ext/fts5/fts5_storage.c 13794781977c9a624eb8bd7b9509de241e405853
F ext/fts5/fts5_storage.c b95fcca70f94656854e7afcfbb9896455f6b034d
F ext/fts5/fts5_tcl.c 664e710e2bbeed505cb91848772ca7538623a67f
F ext/fts5/fts5_tokenize.c 5a0ad46408d09bcda2bf0addb5af42fdb75ebabb
F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9
@ -1270,7 +1270,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 4b3651677e7132c4c45605bc1f216fc08ef31198
R 691df06fdaf9c3542bb10cf702e4a0f8
P 1cd15a1759004d5d321056905dbb6acff20dc7d9
R d21eb6bee3b06e51f22d32d1e0bd7016
U dan
Z 12262406c5f3f18d2ab88add956e21a6
Z e88e77f44b464406d3184a89736eaa7d

2
manifest.uuid
View File

@ -1 +1 @@
1cd15a1759004d5d321056905dbb6acff20dc7d9
17ef5b59f789e9fa35c4f053246d819987fd06f8