diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index 6c69da97b3..7ad9176f4e 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -1056,6 +1056,20 @@ static int fts5SpecialInsert( return rc; } +static int fts5SpecialDelete( + Fts5Table *pTab, + sqlite3_value **apVal, + sqlite3_int64 *piRowid +){ + int rc = SQLITE_OK; + int eType1 = sqlite3_value_type(apVal[1]); + if( eType1==SQLITE_INTEGER ){ + sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); + rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]); + } + return rc; +} + /* ** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be @@ -1086,17 +1100,25 @@ static int fts5UpdateMethod( */ assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) ); - if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){ - return fts5SpecialInsert(pTab, - apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1] - ); + if( nArg>1 ){ + sqlite3_value *pCmd = sqlite3_value_type(apVal[2 + pConfig->nCol]); + if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){ + const char *z = sqlite3_value_text(pCmd); + if( pConfig->bExternalContent && sqlite3_stricmp("delete", z) ){ + return fts5SpecialDelete(pTab, apVal, pRowid); + }else{ + return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]); + } + } } eType0 = sqlite3_value_type(apVal[0]); eConflict = sqlite3_vtab_on_conflict(pConfig->db); assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); - if( eType0==SQLITE_INTEGER ){ + assert( pVtab->zErrMsg==0 ); + + if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); } diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 371c99d745..ef5b9e56c7 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -76,6 +76,9 @@ struct Fts5Config { char **azCol; /* Column names */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ + int bExternalContent; /* Content is external */ + char *zContent; /* "content=" option value (or NULL) */ + char *zContentRowid; /* "content_rowid=" option value (or NULL) */ Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; @@ -410,6 +413,8 @@ int sqlite3Fts5StorageRollback(Fts5Storage *p); int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*); +int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index fd6b051a08..07255c400a 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -192,6 +192,47 @@ static char *fts5TrimString(char *z){ return z; } +/* +** Duplicate the string passed as the only argument into a buffer allocated +** by sqlite3_malloc(). +** +** Return 0 if an OOM error is encountered. +*/ +static char *fts5Strdup(int *pRc, const char *z){ + char *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_mprintf("%s", z); + if( pRet==0 ) *pRc = SQLITE_NOMEM; + } + return pRet; +} + +/* +** Argument z points to a nul-terminated string containing an SQL identifier. +** This function returns a copy of the identifier enclosed in backtick +** quotes. +*/ +static char *fts5EscapeName(int *pRc, const char *z){ + char *pRet = 0; + if( *pRc==SQLITE_OK ){ + int n = strlen(z); + pRet = (char*)sqlite3_malloc(2 * 2*n + 1); + if( pRet==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + int i; + char *p = pRet; + for(i=0; izContent ){ + *pzErr = sqlite3_mprintf("multiple content=... directives"); + rc = SQLITE_ERROR; + }else{ + pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg); + pConfig->bExternalContent = 1; + if( pConfig->zContent==0 ) rc = SQLITE_NOMEM; + } + return rc; + } + + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ + int rc = SQLITE_OK; + if( pConfig->zContentRowid ){ + *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); + rc = SQLITE_ERROR; + }else{ + pConfig->zContentRowid = fts5EscapeName(&rc, zArg); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd); return SQLITE_ERROR; } -/* -** Duplicate the string passed as the only argument into a buffer allocated -** by sqlite3_malloc(). -** -** Return 0 if an OOM error is encountered. -*/ -static char *fts5Strdup(int *pRc, const char *z){ - char *pRet = 0; - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_mprintf("%s", z); - if( pRet==0 ) *pRc = SQLITE_NOMEM; - } - return pRet; -} - /* ** Allocate an instance of the default tokenizer ("simple") at ** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error @@ -422,6 +472,20 @@ int sqlite3Fts5ConfigParse( rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); } + /* If no zContent option was specified, fill in the default values. */ + if( rc==SQLITE_OK && pRet->zContent==0 ){ + pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName); + if( pRet->zContent==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_free(pRet->zContentRowid); + pRet->zContentRowid = 0; + } + } + if( rc==SQLITE_OK && pRet->zContentRowid==0 ){ + pRet->zContentRowid = fts5Strdup(&rc, "rowid"); + } + if( rc!=SQLITE_OK ){ sqlite3Fts5ConfigFree(pRet); *ppOut = 0; @@ -447,6 +511,8 @@ void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ sqlite3_free(pConfig->aPrefix); sqlite3_free(pConfig->zRank); sqlite3_free(pConfig->zRankArgs); + sqlite3_free(pConfig->zContent); + sqlite3_free(pConfig->zContentRowid); sqlite3_free(pConfig); } } diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 830af586b3..0d0c5bae29 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1041,19 +1041,24 @@ i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ ** It is the responsibility of the caller to eventually free the returned ** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. */ -static char *fts5Strndup(const char *pIn, int nIn){ - char *zRet = (char*)sqlite3_malloc(nIn+1); - if( zRet ){ - memcpy(zRet, pIn, nIn); - zRet[nIn] = '\0'; +static char *fts5Strndup(int *pRc, const char *pIn, int nIn){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + zRet = (char*)sqlite3_malloc(nIn+1); + if( zRet ){ + memcpy(zRet, pIn, nIn); + zRet[nIn] = '\0'; + }else{ + *pRc = SQLITE_NOMEM; + } } return zRet; } static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ - *pz = fts5Strndup(pToken->p, pToken->n); - if( *pz==0 ) return SQLITE_NOMEM; - return SQLITE_OK; + int rc = SQLITE_OK; + *pz = fts5Strndup(&rc, pToken->p, pToken->n); + return rc; } /* @@ -1139,6 +1144,7 @@ static int fts5ParseTokenize( int iEnd, /* End offset of token */ int iPos /* Position offset of token */ ){ + int rc = SQLITE_OK; const int SZALLOC = 8; TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; @@ -1159,9 +1165,9 @@ static int fts5ParseTokenize( pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = fts5Strndup(pToken, nToken); + pTerm->zTerm = fts5Strndup(&rc, pToken, nToken); - return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM; + return rc; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 0a9ba0c8ad..0bbf25fdab 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -61,9 +61,9 @@ static int fts5StorageGetStmt( assert( eStmt>=0 && eStmtaStmt) ); if( p->aStmt[eStmt]==0 ){ const char *azStmt[] = { - "SELECT * FROM %Q.'%q_content' ORDER BY id ASC", /* SCAN_ASC */ - "SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */ - "SELECT * FROM %Q.'%q_content' WHERE rowid=?", /* LOOKUP */ + "SELECT * FROM %s ORDER BY id ASC", /* SCAN_ASC */ + "SELECT * FROM %s ORDER BY id DESC", /* SCAN_DESC */ + "SELECT * FROM %s WHERE %s=?", /* LOOKUP */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ @@ -75,32 +75,47 @@ static int fts5StorageGetStmt( "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ }; - Fts5Config *pConfig = p->pConfig; + Fts5Config *pC = p->pConfig; char *zSql = 0; - if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){ - int nCol = pConfig->nCol + 1; - char *zBind; - int i; + switch( eStmt ){ + case FTS5_STMT_SCAN_ASC: + case FTS5_STMT_SCAN_DESC: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent); + break; - zBind = sqlite3_malloc(1 + nCol*2); - if( zBind ){ - for(i=0; izContent, pC->zContentRowid); + break; + + case FTS5_STMT_INSERT_CONTENT: + case FTS5_STMT_REPLACE_CONTENT: { + int nCol = pC->nCol + 1; + char *zBind; + int i; + + zBind = sqlite3_malloc(1 + nCol*2); + if( zBind ){ + for(i=0; izDb, pC->zName, zBind); + sqlite3_free(zBind); } - zBind[i*2-1] = '\0'; - zSql = sqlite3_mprintf(azStmt[eStmt],pConfig->zDb,pConfig->zName,zBind); - sqlite3_free(zBind); + break; } - }else{ - zSql = sqlite3_mprintf(azStmt[eStmt], pConfig->zDb, pConfig->zName); + + default: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); + break; } if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->aStmt[eStmt], 0); + rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0); sqlite3_free(zSql); } } @@ -190,18 +205,21 @@ int sqlite3Fts5StorageOpen( p->pIndex = pIndex; if( bCreate ){ - int i; - char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); - if( zDefn==0 ){ - rc = SQLITE_NOMEM; - }else{ - int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY"); - for(i=0; inCol; i++){ - iOff += sprintf(&zDefn[iOff], ", c%d", i); + if( pConfig->bExternalContent==0 ){ + char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); + if( zDefn==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY"); + for(i=0; inCol; i++){ + iOff += sprintf(&zDefn[iOff], ", c%d", i); + } + rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } - rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); + sqlite3_free(zDefn); } - sqlite3_free(zDefn); + if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr @@ -432,6 +450,78 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){ return rc; } +int sqlite3Fts5StorageSpecialDelete( + Fts5Storage *p, + i64 iDel, + sqlite3_value **apVal +){ + Fts5Config *pConfig = p->pConfig; + int rc; + sqlite3_stmt *pDel; + + assert( p->pConfig->bExternalContent ); + rc = fts5StorageLoadTotals(p, 1); + + /* Delete the index records */ + if( rc==SQLITE_OK ){ + int iCol; + Fts5InsertCtx ctx; + ctx.pStorage = p; + ctx.iCol = -1; + + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel); + for(iCol=0; rc==SQLITE_OK && iColnCol; iCol++){ + rc = sqlite3Fts5Tokenize(pConfig, + (const char*)sqlite3_value_text(apVal[iCol]), + sqlite3_value_bytes(apVal[iCol]), + (void*)&ctx, + fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + } + p->nTotalRow--; + } + + /* Delete the %_docsize record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iDel); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + + return rc; + +} + +/* +** Allocate a new rowid. This is used for "external content" tables when +** a NULL value is inserted into the rowid column. The new rowid is allocated +** by inserting a dummy row into the %_docsize table. The dummy will be +** overwritten later. +*/ +static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ + sqlite3_stmt *pReplace = 0; + int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pReplace, 1); + sqlite3_bind_null(pReplace, 2); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + if( rc==SQLITE_OK ){ + *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); + } + return rc; +} + /* ** Insert a new row into the FTS table. */ @@ -453,27 +543,35 @@ int sqlite3Fts5StorageInsert( rc = fts5StorageLoadTotals(p, 1); /* Insert the new row into the %_content table. */ - if( rc==SQLITE_OK ){ - if( eConflict==SQLITE_REPLACE ){ - eStmt = FTS5_STMT_REPLACE_CONTENT; + if( rc==SQLITE_OK && pConfig->bExternalContent==0 ){ + if( pConfig->bExternalContent ){ if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ - rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); + *piRowid = sqlite3_value_int64(apVal[1]); + }else{ + rc = fts5StorageNewRowid(p, piRowid); } }else{ - eStmt = FTS5_STMT_INSERT_CONTENT; + if( eConflict==SQLITE_REPLACE ){ + eStmt = FTS5_STMT_REPLACE_CONTENT; + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); + } + }else{ + eStmt = FTS5_STMT_INSERT_CONTENT; + } + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, eStmt, &pInsert); + } + for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + rc = sqlite3_bind_value(pInsert, i, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + } + *piRowid = sqlite3_last_insert_rowid(pConfig->db); } } - if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, eStmt, &pInsert); - } - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); - } - if( rc==SQLITE_OK ){ - sqlite3_step(pInsert); - rc = sqlite3_reset(pInsert); - } - *piRowid = sqlite3_last_insert_rowid(pConfig->db); /* Add new entries to the FTS index */ if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index c472027698..90fe542994 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\srank\scolumn\sto\sbe\sremapped\son\sa\sper-query\sbasis\sby\sincluding\sa\sterm\ssimilar\sto\s"rank\smatch\s'bm25(10,2)'"\sin\sa\swhere\sclause. -D 2015-01-02T14:55:22.175 +C Add\ssupport\sfor\sexternal\scontent\stables\sto\sfts5. +D 2015-01-03T20:44:58.134 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,16 +104,16 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786 -F ext/fts5/fts5.c a80283dca24506f1c748fffbba8d87ae4d348b50 +F ext/fts5/fts5.c 16177d7f81af1852cf7f477b5ae119215ad6044a F ext/fts5/fts5.h 4f9d2c477c0ee1907164642471329a82cb6b203b -F ext/fts5/fts5Int.h b5d7970b851d2b4f1745cd2d5c95216c9847aef2 +F ext/fts5/fts5Int.h 8062dc2363c863dc8a5b2e5651cb8c966bd6c4cb F ext/fts5/fts5_aux.c 445e54031ff94174673f4f5aac6c064df20a2a6b F ext/fts5/fts5_buffer.c 1bc5c762bb2e9b4a40b2e8a820a31b809e72eec1 -F ext/fts5/fts5_config.c 74a860e10c5583831f04d0088c4a49a3c6eca43d -F ext/fts5/fts5_expr.c 27d3d2deebae277c34ae2bb3d501dd879c442ba5 +F ext/fts5/fts5_config.c 16d647c7bfe50d4e823267188e12e2d001d655e0 +F ext/fts5/fts5_expr.c 317093f00a2ccdaaee0a5290f9f228c600189c41 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279 F ext/fts5/fts5_index.c 4a8e8535b4303400ddb5f6fb08152da0d88ebf6f -F ext/fts5/fts5_storage.c 13794781977c9a624eb8bd7b9509de241e405853 +F ext/fts5/fts5_storage.c b95fcca70f94656854e7afcfbb9896455f6b034d F ext/fts5/fts5_tcl.c 664e710e2bbeed505cb91848772ca7538623a67f F ext/fts5/fts5_tokenize.c 5a0ad46408d09bcda2bf0addb5af42fdb75ebabb F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9 @@ -1270,7 +1270,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b3651677e7132c4c45605bc1f216fc08ef31198 -R 691df06fdaf9c3542bb10cf702e4a0f8 +P 1cd15a1759004d5d321056905dbb6acff20dc7d9 +R d21eb6bee3b06e51f22d32d1e0bd7016 U dan -Z 12262406c5f3f18d2ab88add956e21a6 +Z e88e77f44b464406d3184a89736eaa7d diff --git a/manifest.uuid b/manifest.uuid index 9ca9c0f833..49cb51dbf0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1cd15a1759004d5d321056905dbb6acff20dc7d9 \ No newline at end of file +17ef5b59f789e9fa35c4f053246d819987fd06f8 \ No newline at end of file