Fixes for the xColumnSize() fts5 extension API.

FossilOrigin-Name: 43fcb844726cfeeb1c8a0dbfaa0d2ca22e6ac16c
This commit is contained in:
dan 2014-07-19 15:35:09 +00:00
parent 30b650e2f4
commit 4a165c0af4
9 changed files with 198 additions and 38 deletions

View File

@ -69,16 +69,30 @@ struct Fts5Cursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
int idxNum; /* idxNum passed to xFilter() */
sqlite3_stmt *pStmt; /* Statement used to read %_content */
int bEof; /* True at EOF */
Fts5Expr *pExpr; /* Expression for MATCH queries */
int bSeekRequired; /* True if seek is required */
int csrflags; /* Mask of cursor flags (see below) */
Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */
/* Variables used by auxiliary functions */
i64 iCsrId; /* Cursor id */
Fts5Auxiliary *pAux; /* Currently executing function */
int *aColumnSize; /* Values for xColumnSize() */
};
/*
** Values for Fts5Cursor.csrflags
*/
#define FTS5CSR_REQUIRE_CONTENT 0x01
#define FTS5CSR_REQUIRE_DOCSIZE 0x02
#define FTS5CSR_EOF 0x04
/*
** Macros to Set(), Clear() and Test() cursor flags.
*/
#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag))
#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag))
/*
** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
** argument is non-zero, attempt delete the shadow tables from teh database
@ -275,13 +289,17 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
*/
static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts5Table *pTab = (Fts5Table*)pVTab;
Fts5Config *pConfig = pTab->pConfig;
Fts5Cursor *pCsr; /* New cursor object */
int nByte; /* Bytes of space to allocate */
int rc = SQLITE_OK; /* Return code */
pCsr = (Fts5Cursor*)sqlite3_malloc(sizeof(Fts5Cursor));
nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
if( pCsr ){
Fts5Global *pGlobal = pTab->pGlobal;
memset(pCsr, 0, sizeof(Fts5Cursor));
memset(pCsr, 0, nByte);
pCsr->aColumnSize = (int*)&pCsr[1];
pCsr->pNext = pGlobal->pCsr;
pGlobal->pCsr = pCsr;
pCsr->iCsrId = ++pGlobal->iNextId;
@ -338,15 +356,17 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
if( ePlan!=FTS5_PLAN_MATCH ){
rc = sqlite3_step(pCsr->pStmt);
if( rc!=SQLITE_ROW ){
pCsr->bEof = 1;
CsrFlagSet(pCsr, FTS5CSR_EOF);
rc = sqlite3_reset(pCsr->pStmt);
}else{
rc = SQLITE_OK;
}
}else{
rc = sqlite3Fts5ExprNext(pCsr->pExpr);
pCsr->bEof = sqlite3Fts5ExprEof(pCsr->pExpr);
pCsr->bSeekRequired = 1;
if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
CsrFlagSet(pCsr, FTS5CSR_EOF);
}
CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
}
return rc;
@ -371,8 +391,10 @@ static int fts5FilterMethod(
int eStmt = fts5StmtType(idxNum);
int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0);
memset(&pCursor[1], 0, sizeof(Fts5Cursor) - sizeof(sqlite3_vtab_cursor));
pCsr->idxNum = idxNum;
assert( pCsr->pStmt==0 );
assert( pCsr->pExpr==0 );
assert( pCsr->csrflags==0 );
rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
if( rc==SQLITE_OK ){
@ -382,8 +404,10 @@ static int fts5FilterMethod(
rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
pCsr->bEof = sqlite3Fts5ExprEof(pCsr->pExpr);
pCsr->bSeekRequired = 1;
if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
CsrFlagSet(pCsr, FTS5CSR_EOF);
}
CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
}
}else{
if( ePlan==FTS5_PLAN_ROWID ){
@ -402,7 +426,7 @@ static int fts5FilterMethod(
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
return pCsr->bEof;
return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
}
/*
@ -415,7 +439,7 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int ePlan = FTS5_PLAN(pCsr->idxNum);
assert( pCsr->bEof==0 );
assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
if( ePlan!=FTS5_PLAN_MATCH ){
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}else{
@ -431,13 +455,14 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
int rc = SQLITE_OK;
if( pCsr->bSeekRequired ){
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
assert( pCsr->pExpr );
sqlite3_reset(pCsr->pStmt);
sqlite3_bind_int64(pCsr->pStmt, 1, sqlite3Fts5ExprRowid(pCsr->pExpr));
rc = sqlite3_step(pCsr->pStmt);
if( rc==SQLITE_ROW ){
rc = SQLITE_OK;
CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
}else{
rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK ){
@ -461,7 +486,7 @@ static int fts5ColumnMethod(
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int rc = SQLITE_OK;
assert( pCsr->bEof==0 );
assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
if( iCol==pConfig->nCol ){
/* User is requesting the value of the special column with the same name
@ -639,8 +664,20 @@ static int fts5ApiColumnText(
}
static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
assert( 0 );
return 0;
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
int rc = SQLITE_OK;
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
}
if( iCol>=0 && iCol<pTab->pConfig->nCol ){
*pnToken = pCsr->aColumnSize[iCol];
}else{
*pnToken = 0;
}
return rc;
}
static int fts5ApiPoslist(

View File

@ -42,9 +42,17 @@ typedef void (*fts5_extension_function)(
);
/*
**
** xUserData:
** Return a copy of the context pointer the extension function was
** registered with.
**
** xColumnCount:
** Returns the number of columns in the FTS5 table.
**
** xColumnSize:
** Reports the size in tokens of a column value from the current row.
**
** xPhraseCount:
** Returns the number of phrases in the current query expression.
**

View File

@ -295,6 +295,9 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol);
/*
** End of interface to code in fts5_storage.c.

View File

@ -47,11 +47,24 @@ static void fts5TestFunction(
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
}
nCol = pApi->xColumnCount(pFts);
if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
nCol = pApi->xColumnCount(pFts);
sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
}
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnsize ");
}
if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnsize") ){
if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
for(i=0; rc==SQLITE_OK && i<nCol; i++){
int colsz = 0;
rc = pApi->xColumnSize(pFts, i, &colsz);
sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
}
if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
}
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasecount ");
}

View File

@ -17,8 +17,7 @@
struct Fts5Storage {
Fts5Config *pConfig;
Fts5Index *pIndex;
sqlite3_stmt *aStmt[8];
sqlite3_stmt *aStmt[9];
};
@ -36,9 +35,11 @@ struct Fts5Storage {
#define FTS5_STMT_REPLACE_CONTENT 4
#define FTS5_STMT_DELETE_CONTENT 5
#define FTS5_STMT_INSERT_DOCSIZE 6
#define FTS5_STMT_REPLACE_DOCSIZE 6
#define FTS5_STMT_DELETE_DOCSIZE 7
#define FTS5_STMT_LOOKUP_DOCSIZE 8
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
@ -62,8 +63,10 @@ static int fts5StorageGetStmt(
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
"DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
"INSERT INTO %Q.'%q_docsize' VALUES(?,?)", /* INSERT_DOCSIZE */
"REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */
"DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
"SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
};
Fts5Config *pConfig = p->pConfig;
char *zSql = 0;
@ -234,6 +237,7 @@ typedef struct Fts5InsertCtx Fts5InsertCtx;
struct Fts5InsertCtx {
Fts5Storage *pStorage;
int iCol;
int szCol; /* Size of column value in tokens */
};
/*
@ -249,6 +253,7 @@ static int fts5StorageInsertCallback(
){
Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
Fts5Index *pIdx = pCtx->pStorage->pIndex;
pCtx->szCol = iPos+1;
sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
return SQLITE_OK;
}
@ -289,6 +294,27 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
return rc;
}
/*
** Insert a record into the %_docsize table. Specifically, do:
**
** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
*/
static int fts5StorageInsertDocsize(
Fts5Storage *p, /* Storage module to write to */
i64 iRowid, /* id value */
Fts5Buffer *pBuf /* sz value */
){
sqlite3_stmt *pReplace = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pReplace, 1, iRowid);
sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
}
return rc;
}
/*
** Insert a new row into the FTS table.
*/
@ -304,6 +330,9 @@ int sqlite3Fts5StorageInsert(
int eStmt; /* Type of statement used on %_content */
int i; /* Counter variable */
Fts5InsertCtx ctx; /* Tokenization callback context object */
Fts5Buffer buf; /* Buffer used to build up %_docsize blob */
memset(&buf, 0, sizeof(Fts5Buffer));
/* Insert the new row into the %_content table. */
if( eConflict==SQLITE_REPLACE ){
@ -330,14 +359,22 @@ int sqlite3Fts5StorageInsert(
sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
ctx.pStorage = p;
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
(const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
sqlite3_value_bytes(apVal[ctx.iCol+2]),
(void*)&ctx,
fts5StorageInsertCallback
);
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
}
/* Write the %_docsize record */
if( rc==SQLITE_OK ){
rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
}
sqlite3_free(buf.p);
return rc;
}
@ -458,4 +495,50 @@ void sqlite3Fts5StorageStmtRelease(
}
}
static int fts5StorageDecodeSizeArray(
int *aCol, int nCol, /* Array to populate */
const u8 *aBlob, int nBlob /* Record to read varints from */
){
int i;
int iOff = 0;
for(i=0; i<nCol; i++){
if( iOff>=nBlob ) return 1;
iOff += getVarint32(&aBlob[iOff], aCol[i]);
}
return (iOff!=nBlob);
}
/*
** Argument aCol points to an array of integers containing one entry for
** each table column. This function reads the %_docsize record for the
** specified rowid and populates aCol[] with the results.
**
** An SQLite error code is returned if an error occurs, or SQLITE_OK
** otherwise.
*/
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
int nCol = p->pConfig->nCol;
sqlite3_stmt *pLookup = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup);
if( rc==SQLITE_OK ){
int bCorrupt = 1;
sqlite3_bind_int64(pLookup, 1, iRowid);
if( SQLITE_ROW==sqlite3_step(pLookup) ){
const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
int nBlob = sqlite3_column_bytes(pLookup, 0);
if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
bCorrupt = 0;
}
}
rc = sqlite3_reset(pLookup);
if( bCorrupt && rc==SQLITE_OK ){
rc = SQLITE_CORRUPT_VTAB;
}
}
return rc;
}
int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){
return 0;
}

View File

@ -1,5 +1,5 @@
C Fix\sissues\swith\sposition\slists\sand\sNEAR\sconstraints.
D 2014-07-18T19:59:00.547
C Fixes\sfor\sthe\sxColumnSize()\sfts5\sextension\sAPI.
D 2014-07-19T15:35:09.453
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
F ext/fts5/fts5.c 20bcb1e10756c72b550947236960edf96929ca2f
F ext/fts5/fts5.h cda3b9d73e6ffa6d0cd35b7da6b808bf3a1ada32
F ext/fts5/fts5Int.h 6cf315d3999c14572012d676fa1baf4f4323587b
F ext/fts5/fts5_aux.c 27b082732fd76277fd7e9277f52903723d97f99b
F ext/fts5/fts5.c 86655d1e2ba35c719d3cc480cb9fdd7f3887b74e
F ext/fts5/fts5.h 844898034fa3e0458d93a6c34dd6ba6bd3c7e03a
F ext/fts5/fts5Int.h cca221a5cf7234f92faf3b4b5f2e4cf43bce83ee
F ext/fts5/fts5_aux.c 978a90fe90a6d34d9bd260948b5678caf5489894
F ext/fts5/fts5_buffer.c 71cf2016b2881e7aea39f952995eafa510d96cbd
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36
F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8
F ext/fts5/fts5_storage.c fcf66173e55927cee0675ecfb1038d0000e4fa10
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
@ -599,7 +599,7 @@ F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343
F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
F test/fts5ae.test 5d5ffba68e850d9ade99cdd3f5c6431c82dad81d
F test/fts5ae.test 7da37ac01debf2e238552d0ef2f61669fd232936
F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@ -767,7 +767,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
F test/permutations.test 0b5333e5dcdeffba0ecbe5ee8dc7577029ffab6c
F test/permutations.test c3eb62a88337d9a5046c509dd90ba6d43debc76d
F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@ -1195,7 +1195,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 5808f30fae0d844c52a785bf18872be371d4af68
R c645036fa73431553c03d7990bbe09ec
P 16352d3654d5672cd0251db51dbe19f779373feb
R a0466df485f1c616be1f64f0989b7925
U dan
Z 973aba4d2c5c8ae6a3e94f9739fdfe1b
Z affe87335b53c4c8634348527de72153

View File

@ -1 +1 @@
16352d3654d5672cd0251db51dbe19f779373feb
43fcb844726cfeeb1c8a0dbfaa0d2ca22e6ac16c

View File

@ -121,16 +121,32 @@ do_execsql_test 4.0 {
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}
breakpoint
do_execsql_test 4.1 {
SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
1 {0.5 {} {}}
}
#93 {0.5 1.6 {}}
#-------------------------------------------------------------------------
# Test that the xColumnSize() API works.
#
reset_db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE t5 USING fts5(x, y);
INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
INSERT INTO t5 VALUES('', 'a');
INSERT INTO t5 VALUES('a', '');
}
do_execsql_test 5.2 {
SELECT rowid, fts5_test(t5, 'columnsize') FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {1 0}
2 {0 1}
1 {4 6}
}
finish_test

View File

@ -225,7 +225,7 @@ test_suite "fts3" -prefix "" -description {
test_suite "fts5" -prefix "" -description {
All FTS5 tests.
} -files {
fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ea.test
fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
}
test_suite "nofaultsim" -prefix "" -description {