diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index 928ad6d916..b37f72d58d 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -529,46 +529,48 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){ ** on the xClose method of the virtual table interface. */ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); - Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; - Fts5Cursor **pp; - Fts5Auxdata *pData; - Fts5Auxdata *pNext; + if( pCursor ){ + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + Fts5Cursor **pp; + Fts5Auxdata *pData; + Fts5Auxdata *pNext; - fts5CsrNewrow(pCsr); - if( pCsr->pStmt ){ - int eStmt = fts5StmtType(pCsr->idxNum); - sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); - } - if( pCsr->pSorter ){ - Fts5Sorter *pSorter = pCsr->pSorter; - sqlite3_finalize(pSorter->pStmt); - sqlite3_free(pSorter); - } - - if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){ - sqlite3Fts5ExprFree(pCsr->pExpr); - } + fts5CsrNewrow(pCsr); + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr->idxNum); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + if( pCsr->pSorter ){ + Fts5Sorter *pSorter = pCsr->pSorter; + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + } - for(pData=pCsr->pAuxdata; pData; pData=pNext){ - pNext = pData->pNext; - if( pData->xDelete ) pData->xDelete(pData->pPtr); - sqlite3_free(pData); + if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){ + sqlite3Fts5ExprFree(pCsr->pExpr); + } + + for(pData=pCsr->pAuxdata; pData; pData=pNext){ + pNext = pData->pNext; + if( pData->xDelete ) pData->xDelete(pData->pPtr); + sqlite3_free(pData); + } + + /* Remove the cursor from the Fts5Global.pCsr list */ + for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); + *pp = pCsr->pNext; + + sqlite3_finalize(pCsr->pRankArgStmt); + sqlite3_free(pCsr->apRankArg); + + sqlite3_free(pCsr->zSpecial); + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ + sqlite3_free(pCsr->zRank); + sqlite3_free(pCsr->zRankArgs); + } + sqlite3_free(pCsr); } - - /* Remove the cursor from the Fts5Global.pCsr list */ - for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); - *pp = pCsr->pNext; - - sqlite3_finalize(pCsr->pRankArgStmt); - sqlite3_free(pCsr->apRankArg); - - sqlite3_free(pCsr->zSpecial); - if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ - sqlite3_free(pCsr->zRank); - sqlite3_free(pCsr->zRankArgs); - } - sqlite3_free(pCsr); return SQLITE_OK; } @@ -893,7 +895,11 @@ static int fts5CursorParseRank( char *zRank = 0; char *zRankArgs = 0; - rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + if( z==0 ){ + if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + } if( rc==SQLITE_OK ){ pCsr->zRank = zRank; pCsr->zRankArgs = zRankArgs; @@ -1207,16 +1213,16 @@ static int fts5UpdateMethod( ** 3. Values for each of the nCol matchable columns. ** 4. Values for the two hidden columns ( and "rank"). */ - assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) ); eType0 = sqlite3_value_type(apVal[0]); eConflict = sqlite3_vtab_on_conflict(pConfig->db); assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); assert( pVtab->zErrMsg==0 ); + assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) ); fts5TripCursors(pTab); - if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){ + if( eType0==SQLITE_INTEGER ){ if( fts5IsContentless(pTab) ){ pTab->base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", @@ -1227,7 +1233,8 @@ static int fts5UpdateMethod( i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); } - }else if( nArg>1 ){ + }else{ + assert( nArg>1 ); sqlite3_value *pCmd = apVal[2 + pConfig->nCol]; if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){ const char *z = (const char*)sqlite3_value_text(pCmd); @@ -1471,6 +1478,7 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); } if( iCol<0 ){ int i; @@ -1879,6 +1887,23 @@ static int fts5CreateTokenizer( return rc; } +static Fts5TokenizerModule *fts5LocateTokenizer( + Fts5Global *pGlobal, + const char *zName +){ + Fts5TokenizerModule *pMod = 0; + + if( zName==0 ){ + pMod = pGlobal->pDfltTok; + }else{ + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ + if( sqlite3_stricmp(zName, pMod->zName)==0 ) break; + } + } + + return pMod; +} + /* ** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. @@ -1889,21 +1914,13 @@ static int fts5FindTokenizer( void **ppUserData, fts5_tokenizer *pTokenizer /* Populate this object */ ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; int rc = SQLITE_OK; - Fts5TokenizerModule *pTok; + Fts5TokenizerModule *pMod; - if( zName==0 ){ - pTok = pGlobal->pDfltTok; - }else{ - for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){ - if( sqlite3_stricmp(zName, pTok->zName)==0 ) break; - } - } - - if( pTok ){ - *pTokenizer = pTok->x; - *ppUserData = pTok->pUserData; + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + *pTokenizer = pMod->x; + *ppUserData = pMod->pUserData; }else{ memset(pTokenizer, 0, sizeof(fts5_tokenizer)); rc = SQLITE_ERROR; @@ -1917,24 +1934,23 @@ int sqlite3Fts5GetTokenizer( const char **azArg, int nArg, Fts5Tokenizer **ppTok, - fts5_tokenizer **ppTokApi + fts5_tokenizer **ppTokApi, + char **pzErr ){ - Fts5TokenizerModule *pMod = 0; + Fts5TokenizerModule *pMod; int rc = SQLITE_OK; - if( nArg==0 ){ - pMod = pGlobal->pDfltTok; - }else{ - for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ - if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break; - } - } - + pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ + assert( nArg>0 ); rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok); *ppTokApi = &pMod->x; + if( rc!=SQLITE_OK && pzErr ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } } if( rc!=SQLITE_OK ){ diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index d3bdbadc36..34190bdec1 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -61,7 +61,8 @@ int sqlite3Fts5GetTokenizer( const char **azArg, int nArg, Fts5Tokenizer**, - fts5_tokenizer** + fts5_tokenizer**, + char **pzErr ); /* diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 8e4beffe67..4ef9e96070 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -230,11 +230,12 @@ static void fts5HighlightFunction( if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); } sqlite3_free(ctx.zOut); } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } } /* ** End of highlight() implementation. diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 0c79cdfc03..eae18dce27 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -330,11 +330,9 @@ static int fts5ConfigParseSpecial( rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi + (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi, + pzErr ); - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("error in tokenizer constructor"); - } } } } @@ -387,7 +385,7 @@ static int fts5ConfigParseSpecial( static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); return sqlite3Fts5GetTokenizer( - pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi + pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0 ); } @@ -563,7 +561,7 @@ int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; }else{ if( bOption ){ - rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr); + rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr); }else{ rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr); zOne = 0; diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 6463c64c3d..a638559310 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -280,15 +280,15 @@ int sqlite3Fts5ExprPhraseExpr( Fts5ExprPhrase *pCopy; pOrig = pExpr->apExprPhrase[iPhrase]; + pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm + ); pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr)); apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*)); pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode)); pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) ); - pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm - ); for(i=0; rc==SQLITE_OK && inTerm; i++){ pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm); diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 568d816ff1..431533afcc 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -36,6 +36,9 @@ int sqlite3_fts5_may_be_corrupt = 0; ** can extract the sqlite3* pointer from an existing Tcl SQLite ** connection. */ + +extern const char *sqlite3ErrName(int); + struct SqliteDb { sqlite3 *db; }; @@ -390,7 +393,7 @@ static int xF5tApi( #undef CASE if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in api call", 0); + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_ERROR; } @@ -727,8 +730,6 @@ static int f5tTokenizerTokenize( return rc; } -extern const char *sqlite3ErrName(int); - /* ** sqlite3_fts5_token TEXT START END POS */ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 3f4261c698..428a637e38 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -529,11 +529,16 @@ static int fts5PorterCreate( int rc = SQLITE_OK; PorterTokenizer *pRet; void *pUserdata = 0; + const char *zBase = "unicode61"; + + if( nArg>0 ){ + zBase = azArg[0]; + } pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, "unicode61", &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); }else{ rc = SQLITE_NOMEM; } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 9c56790dc5..ce4c90b9a1 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -297,7 +297,7 @@ do_catchsql_test 12.2 { } {1 {unknown special query: stuff}} do_test 12.3 { - set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] + set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] string is integer $res } {1} diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index 36402d6f6b..522f44ce23 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -268,6 +268,14 @@ do_execsql_test 4.3.3 { {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 } +do_catchsql_test 4.4.3 { + SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' +} {1 {no such function: xyz}} +do_catchsql_test 4.4.4 { + SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL +} {1 {parse error in rank function: }} + + finish_test diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test new file mode 100644 index 0000000000..b5cbc6e9a3 --- /dev/null +++ b/ext/fts5/test/fts5aux.test @@ -0,0 +1,57 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the auxiliary function APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aux + +proc inst {cmd i} { + $cmd xInst $i +} +sqlite3_fts5_create_function db inst inst + +proc colsize {cmd i} { + $cmd xColumnSize $i +} +sqlite3_fts5_create_function db colsize colsize + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1 VALUES('one two', 'two one zero'); +} + +do_catchsql_test 1.1 { + SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} +do_catchsql_test 1.2 { + SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two'; +} {0 {{0 0 1}}} +do_catchsql_test 1.3 { + SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two'; +} {0 {{0 1 0}}} +do_catchsql_test 1.4 { + SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} + +do_catchsql_test 2.1 { + SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} + +do_execsql_test 2.2 { + SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero'; +} {2 3} + + + +finish_test + diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index be2b56a76d..105e11224b 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -215,6 +215,24 @@ do_execsql_test 4.7 { do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {} do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40} +#------------------------------------------------------------------------- +# Test that if the 'rowid' field of a 'delete' is not an integer, no +# changes are made to the FTS index. +# +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=); + INSERT INTO t5(rowid, a, b) VALUES(-1, 'one', 'two'); + INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four'); + INSERT INTO t5(rowid, a, b) VALUES( 1, 'five', 'six'); +} + +set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}] + +do_execsql_test 5.1 { + INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four'); + SELECT md5sum(id, block) FROM t5_data; +} $::checksum + finish_test diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 0791ab0cf9..7d0ea9d2bc 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -70,7 +70,24 @@ do_test 2.1 { } {} #-------------------------------------------------------------------- +# A mundane test - missing row in the %_content table. # +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(x); + INSERT INTO t3 VALUES('one o'); + INSERT INTO t3 VALUES('two e'); + INSERT INTO t3 VALUES('three o'); + INSERT INTO t3 VALUES('four e'); + INSERT INTO t3 VALUES('five o'); +} +do_execsql_test 3.1 { + SELECT * FROM t3 WHERE t3 MATCH 'o' +} {{one o} {three o} {five o}} + +do_catchsql_test 3.1 { + DELETE FROM t3_content WHERE rowid = 3; + SELECT * FROM t3 WHERE t3 MATCH 'o'; +} {1 {database disk image is malformed}} finish_test diff --git a/ext/fts5/test/fts5doclist.test b/ext/fts5/test/fts5doclist.test new file mode 100644 index 0000000000..eb1dc9a695 --- /dev/null +++ b/ext/fts5/test/fts5doclist.test @@ -0,0 +1,41 @@ +# 2015 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on edge cases in the doclist format. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5doclist + + +#------------------------------------------------------------------------- +# Create a table with 1000 columns. Then add some large documents to it. +# All text is in the right most column of the table. +# +do_test 1.0 { + set cols [list] + for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" } + execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])" +} {} + +db func rnddoc fts5_rnddoc +do_execsql_test 1.1 { + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) + INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii; +} + +do_execsql_test 1.2 { + INSERT INTO ccc(ccc) VALUES('integrity-check'); +} + + +finish_test + diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index de79900bb6..d7ced10c72 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +if 1 { + #------------------------------------------------------------------------- # An OOM while dropping an fts5 table. # @@ -40,6 +42,154 @@ do_faultsim_test 1 -faults oom-* -prep { faultsim_test_result [list 0 {}] } +#------------------------------------------------------------------------- +# An OOM within an "ORDER BY rank" query. +# +db func rnddoc fts5_rnddoc +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xx USING fts5(x); + INSERT INTO xx VALUES ('abc ' || rnddoc(10)); + INSERT INTO xx VALUES ('abc abc' || rnddoc(9)); + INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8)); +} {} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM xx } +} -body { + execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank } +} -test { + faultsim_test_result [list 0 {3 2 1}] +} + +#------------------------------------------------------------------------- +# An OOM while "reseeking" an FTS cursor. +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE jj USING fts5(j); + INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s'); + INSERT INTO jj(rowid, j) VALUES(202, 't w t f s'); + INSERT INTO jj(rowid, j) VALUES(303, 'w t f'); + INSERT INTO jj(rowid, j) VALUES(404, 't'); +} +faultsim_save_and_close + +do_faultsim_test 3 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM jj } +} -body { + set res [list] + db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } { + lappend res $rowid + if {$rowid==303} { + execsql { DELETE FROM jj WHERE rowid=404 } + } + } + set res +} -test { + faultsim_test_result [list 0 {101 202 303}] +} + +#------------------------------------------------------------------------- +# An OOM within a special "*reads" query. +# +reset_db +db func rnddoc fts5_rnddoc +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + + WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 ) + INSERT INTO x1 SELECT rnddoc(5) FROM ii; +} + +set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] + +do_faultsim_test 4 -faults oom-* -body { + db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} +} -test { + faultsim_test_result {0 {0 {} 3}} +} + +#------------------------------------------------------------------------- +# An OOM within a query that uses a custom rank function. +# +reset_db +do_execsql_test 5.0 { + PRAGMA encoding='utf16'; + CREATE VIRTUAL TABLE x2 USING fts5(x); + INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3 + INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6 + INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2 + INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5 + INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1 +} + +proc rowidmod {cmd mod} { + set row [$cmd xRowid] + expr {$row % $mod} +} +sqlite3_fts5_create_function db rowidmod rowidmod + +do_faultsim_test 5.1 -faults oom-* -body { + db eval { + SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND + rank MATCH "rowidmod('7')" ORDER BY rank + } +} -test { + faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}} +} + +proc rowidprefix {cmd prefix} { + set row [$cmd xRowid] + set {} "${row}-${prefix}" +} +sqlite3_fts5_create_function db rowidprefix rowidprefix + +set str [string repeat abcdefghijklmnopqrstuvwxyz 10] +do_faultsim_test 5.2 -faults oom-* -body { + db eval " + SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND + rank MATCH 'rowidprefix(''$::str'')' + LIMIT 1 + " +} -test { + faultsim_test_result "0 {10-$::str {a b c}}" +} + +} + + +#------------------------------------------------------------------------- +# OOM errors within auxiliary functions. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x3 USING fts5(xxx); + INSERT INTO x3 VALUES('a b c d c b a'); +} + +do_faultsim_test 6.1 -faults oom-t* -body { + db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' } +} -test { + faultsim_test_result {0 {{a b *c* d *c* b a}}} +} + +proc firstinst {cmd} { + foreach {p c o} [$cmd xInst 0] {} + expr $c*100 + $o +} +sqlite3_fts5_create_function db firstinst firstinst + +do_faultsim_test 6.2 -faults oom-t* -body { + db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' } +} -test { + faultsim_test_result {0 2} {1 SQLITE_NOMEM} +} + + + finish_test diff --git a/ext/fts5/test/fts5plan.test b/ext/fts5/test/fts5plan.test new file mode 100644 index 0000000000..1670f89faa --- /dev/null +++ b/ext/fts5/test/fts5plan.test @@ -0,0 +1,61 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing the planner (xBestIndex function). +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5plan + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + CREATE VIRTUAL TABLE f1 USING fts5(ff); +} + +do_eqp_test 1.1 { + SELECT * FROM t1, f1 WHERE f1 MATCH t1.x +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} +} + +do_eqp_test 1.2 { + SELECT * FROM t1, f1 WHERE f1 > t1.x +} { + 0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 1 0 {SCAN TABLE t1} +} + +do_eqp_test 1.3 { + SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff +} { + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + +do_eqp_test 1.4 { + SELECT * FROM f1 ORDER BY rank +} { + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + +do_eqp_test 1.5 { + SELECT * FROM f1 WHERE rank MATCH ? +} { + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} +} + + + + +finish_test + diff --git a/ext/fts5/test/fts5rank.test b/ext/fts5/test/fts5rank.test new file mode 100644 index 0000000000..6da24adee2 --- /dev/null +++ b/ext/fts5/test/fts5rank.test @@ -0,0 +1,39 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing queries that use the "rank" column. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5rank + + +#------------------------------------------------------------------------- +# "ORDER BY rank" + highlight() + large poslists. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xyz USING fts5(z); +} +do_test 1.1 { + set doc [string trim [string repeat "x y " 500]] + execsql { INSERT INTO xyz VALUES($doc) } +} {} +do_execsql_test 1.2 { + SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank +} [list [string map {x [x]} $doc]] + +do_execsql_test 1.3 { + SELECT highlight(xyz, 0, '[', ']') FROM xyz + WHERE xyz MATCH 'x AND y' ORDER BY rank +} [list [string map {x [x] y [y]} $doc]] + +finish_test + diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test index 644a674942..96961792f5 100644 --- a/ext/fts5/test/fts5rebuild.test +++ b/ext/fts5/test/fts5rebuild.test @@ -46,5 +46,16 @@ do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('integrity-check'); } {} + +#------------------------------------------------------------------------- +# Check that 'rebuild' may not be used with a contentless table. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE nc USING fts5(doc, content=); +} + +do_catchsql_test 2.2 { + INSERT INTO nc(nc) VALUES('rebuild'); +} {1 {'rebuild' may not be used with a contentless fts5 table}} finish_test diff --git a/ext/fts5/test/fts5restart.test b/ext/fts5/test/fts5restart.test index 3b1f334221..8fa3600ab3 100644 --- a/ext/fts5/test/fts5restart.test +++ b/ext/fts5/test/fts5restart.test @@ -19,6 +19,10 @@ do_execsql_test 1.0 { CREATE VIRTUAL TABLE f1 USING fts5(ff); } +#------------------------------------------------------------------------- +# Run the 'optimize' command. Check that it does not disturb ongoing +# full-text queries. +# do_test 1.1 { for {set i 1} {$i < 1000} {incr i} { execsql { INSERT INTO f1 VALUES('a b c d e') } @@ -30,7 +34,6 @@ do_execsql_test 1.2 { SELECT rowid FROM f1 WHERE f1 MATCH 'c'; } $lRowid -breakpoint do_test 1.3 { set res [list] db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } { @@ -42,6 +45,101 @@ do_test 1.3 { set res } $lRowid +do_test 1.4.1 { + sqlite3 db2 test.db + set res [list] + db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } { + if {$rowid == 100} { + set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }] + } + lappend res $rowid + } + set res +} $lRowid + +do_test 1.4.2 { + db2 close + set cres +} {1 {database is locked}} + +#------------------------------------------------------------------------- +# Open a couple of cursors. Then close them in the same order. +# +do_test 2.1 { + set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X] + set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X] + + sqlite3_step $::s1 +} {SQLITE_ROW} +do_test 2.2 { + sqlite3_step $::s2 +} {SQLITE_ROW} + +do_test 2.1 { + sqlite3_finalize $::s1 + sqlite3_finalize $::s2 +} {SQLITE_OK} + +#------------------------------------------------------------------------- +# Copy data between two FTS5 tables. +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE f2 USING fts5(gg); + INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d'; +} +do_execsql_test 3.2 { + SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e' +} $lRowid + +#------------------------------------------------------------------------- +# Remove the row that an FTS5 cursor is currently pointing to. And +# various other similar things. Check that this does not disturb +# ongoing scans. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE n4 USING fts5(n); + INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5'); + INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4'); + INSERT INTO n4(rowid, n) VALUES(300, '2 3 4'); + INSERT INTO n4(rowid, n) VALUES(400, '2 3'); + INSERT INTO n4(rowid, n) VALUES(500, '3'); +} + +do_test 4.1 { + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } { + if {$rowid==300} { + execsql { DELETE FROM n4 WHERE rowid=300 } + } + lappend res $rowid + } + set res +} {100 200 300 400 500} + +do_test 4.2 { + execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') } + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} { + if {$rowid==300} { + execsql { DELETE FROM n4 WHERE rowid=300 } + } + lappend res $rowid + } + set res +} {500 400 300 200 100} + +do_test 4.3 { + execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') } + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} { + if {$rowid==300} { + execsql { DELETE FROM n4 } + } + lappend res $rowid + } + set res +} {500 400 300} + finish_test diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 44de1690fe..42f5956492 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -37,6 +37,14 @@ do_execsql_test 1.4 { DROP TABLE ft1; } +do_catchsql_test 1.5 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch'); +} {1 {no such tokenizer: nosuch}} + +do_catchsql_test 1.6 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch'); +} {1 {error in tokenizer constructor}} + do_execsql_test 2.0 { CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter); INSERT INTO ft1 VALUES('embedded databases'); diff --git a/manifest b/manifest index 370597381b..199e150b25 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sfts5\sbug\sin\shandling\swrites\swhile\sthere\sare\sactive\scursors. -D 2015-04-28T20:24:50.023 +C Improve\sfts5\stests. +D 2015-04-29T20:54:08.849 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,23 +104,23 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a -F ext/fts5/fts5.c 45e82b584bb3fce22c82d8521bccaa2a6e9202fe +F ext/fts5/fts5.c 932284a253cc9bb32caf047879669720680be2c3 F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a -F ext/fts5/fts5Int.h f573fe6c50471f1d66682fce282da801009c54e1 -F ext/fts5/fts5_aux.c fcea18b1a2a3f95a498b52aba2983557d7678a22 +F ext/fts5/fts5Int.h 2e0a1a6b77e1e014b7e9b1479ca686ff79930457 +F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971 F ext/fts5/fts5_buffer.c 8c8cfe7f09ca2767ab53ea883f9a0af0edb6bbae -F ext/fts5/fts5_config.c f344ffa24d2add70fd5bde2b73c44846ad7a06bd -F ext/fts5/fts5_expr.c e647a2626af5c80a6325532b23eea6d1eb252a78 +F ext/fts5/fts5_config.c ecbbd5163758a958106867051892e0dfecf68b5c +F ext/fts5/fts5_expr.c 663c75dfdb1bfd8809d696357d7b55f507815098 F ext/fts5/fts5_hash.c 29d8b0668727863cc1f1efa65efe4dd78635b016 F ext/fts5/fts5_index.c de588982b0237b1605d6c37afd115b34c95c3da1 F ext/fts5/fts5_storage.c ef60fc9dcc4e274f9589165e26833173c273ae18 -F ext/fts5/fts5_tcl.c af1d37fa93bcabc926aa4e89500adedbbe84a520 -F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b +F ext/fts5/fts5_tcl.c 19ab8cfa642950648968dcf25075d6d969900524 +F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32 -F ext/fts5/test/fts5aa.test 87b2e01084c3e2a960ae1500dd5f0729dac2166c +F ext/fts5/test/fts5aa.test e17657bd749cb3982745ec503ce22372dee7340d F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d @@ -131,30 +131,34 @@ F ext/fts5/test/fts5ah.test d74cf8b7de5b8424f732acef69fe12122a12f2bf F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 -F ext/fts5/test/fts5al.test 6a5717faaf7f1e0e866360022d284903f3a4eede +F ext/fts5/test/fts5al.test e6bddd2c11c0d1e3ae189ee51081899d2f4ea570 +F ext/fts5/test/fts5aux.test 1e475d928a3d1decf74167394db20330d7beeb0f F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b -F ext/fts5/test/fts5content.test c4f5b0fe1bc7523bb6706591d05d1194a0aec452 -F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727 +F ext/fts5/test/fts5content.test 532e15b541254410adc7bfb51f94631cfe82de8f +F ext/fts5/test/fts5corrupt.test 138aecc75c36c3dac9259c7f57c5bc3d009255f8 F ext/fts5/test/fts5corrupt2.test 494111fd4f2dab36499cf97718eaba1f7c11e9d0 F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f +F ext/fts5/test/fts5doclist.test 635b80ac785627841a59c583bac702b55d49fdc5 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e F ext/fts5/test/fts5fault1.test ed71717a479bef32d05f02d9c48691011d160d4d F ext/fts5/test/fts5fault2.test 26c3d70648f691e2cc9391e14bbc11a973656383 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 -F ext/fts5/test/fts5fault4.test e860e0cf7e56f2f87330023be1f1ced44128d5c8 +F ext/fts5/test/fts5fault4.test aea710bbf5680ed41afb9d3313c297d429f9feac F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d F ext/fts5/test/fts5hash.test adb7b0442cc1c77c507f07e16d11490486e75dfa F ext/fts5/test/fts5merge.test 453a0717881aa7784885217b2040f3f275caff03 F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54 +F ext/fts5/test/fts5plan.test 89783f70dab89ff936ed6f21d88959b49c853a47 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e F ext/fts5/test/fts5prefix.test 1287803c3df0e43f536196256fb9e0e6baccb4f1 -F ext/fts5/test/fts5rebuild.test ee6792715c6c528cc188e7869d67c3c655889ddb -F ext/fts5/test/fts5restart.test 4934233dd2633665198d563c79f1015e3a2b9518 +F ext/fts5/test/fts5rank.test f59a6b20ec8e08cb130d833dcece59cf9cd92890 +F ext/fts5/test/fts5rebuild.test 77c6613aa048f38b4a12ddfacb2e6e1342e1b066 +F ext/fts5/test/fts5restart.test cd58a5fb552ac10db549482698e503f82693bcd0 F ext/fts5/test/fts5rowid.test a1b2a6d76648c734c1aab11ee1a619067e8d90e6 -F ext/fts5/test/fts5tokenizer.test 7a6ee24db908c09a0dc1eba634ffa17afcc05d86 +F ext/fts5/test/fts5tokenizer.test bbcde2a7473dcaa9a1fc6809aa8965acb7b846ff F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee F ext/fts5/test/fts5unindexed.test f388605341a476b6ab622b4c267cd168f59a5944 @@ -1311,7 +1315,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8e8136f2dc08082c2984462719d9cba0f212c92a -R 65017a9d42efb9cfacfe6e269e92cdbd +P 07f70955392697556ca2951c9b6c3a5204cd5ec0 +R 07c0e8a626b67779b0890a98070e21c4 U dan -Z 91e05f6f4d01e29ed38305d1f49be921 +Z 5236003ea42b936c8f2a3578dc6f6593 diff --git a/manifest.uuid b/manifest.uuid index e0e7db8c5b..615ae11606 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07f70955392697556ca2951c9b6c3a5204cd5ec0 \ No newline at end of file +c1f07a3aa98eac87e2747527d15e5e5562221ceb \ No newline at end of file