mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve fts5 tests.

Browse Source 
FossilOrigin-Name: c1f07a3aa98eac87e2747527d15e5e5562221ceb
This commit is contained in:
dan 2015-04-29 20:54:08 +00:00
parent 90dd70226f
commit 7b2ec1ae41
21 changed files with 637 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
ext/fts5/fts5.c
View File

@ -529,46 +529,48 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
Fts5Cursor **pp;
Fts5Auxdata *pData;
Fts5Auxdata *pNext;
if( pCursor ){
Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
Fts5Cursor **pp;
Fts5Auxdata *pData;
Fts5Auxdata *pNext;
fts5CsrNewrow(pCsr);
if( pCsr->pStmt ){
int eStmt = fts5StmtType(pCsr->idxNum);
sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
}
if( pCsr->pSorter ){
Fts5Sorter *pSorter = pCsr->pSorter;
sqlite3_finalize(pSorter->pStmt);
sqlite3_free(pSorter);
}
if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
sqlite3Fts5ExprFree(pCsr->pExpr);
}
fts5CsrNewrow(pCsr);
if( pCsr->pStmt ){
int eStmt = fts5StmtType(pCsr->idxNum);
sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
}
if( pCsr->pSorter ){
Fts5Sorter *pSorter = pCsr->pSorter;
sqlite3_finalize(pSorter->pStmt);
sqlite3_free(pSorter);
}
for(pData=pCsr->pAuxdata; pData; pData=pNext){
pNext = pData->pNext;
if( pData->xDelete ) pData->xDelete(pData->pPtr);
sqlite3_free(pData);
if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
sqlite3Fts5ExprFree(pCsr->pExpr);
}
for(pData=pCsr->pAuxdata; pData; pData=pNext){
pNext = pData->pNext;
if( pData->xDelete ) pData->xDelete(pData->pPtr);
sqlite3_free(pData);
}
/* Remove the cursor from the Fts5Global.pCsr list */
for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
*pp = pCsr->pNext;
sqlite3_finalize(pCsr->pRankArgStmt);
sqlite3_free(pCsr->apRankArg);
sqlite3_free(pCsr->zSpecial);
if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
sqlite3_free(pCsr->zRank);
sqlite3_free(pCsr->zRankArgs);
}
sqlite3_free(pCsr);
}
/* Remove the cursor from the Fts5Global.pCsr list */
for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
*pp = pCsr->pNext;
sqlite3_finalize(pCsr->pRankArgStmt);
sqlite3_free(pCsr->apRankArg);
sqlite3_free(pCsr->zSpecial);
if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
sqlite3_free(pCsr->zRank);
sqlite3_free(pCsr->zRankArgs);
}
sqlite3_free(pCsr);
return SQLITE_OK;
}
@ -893,7 +895,11 @@ static int fts5CursorParseRank(
char *zRank = 0;
char *zRankArgs = 0;
rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
if( z==0 ){
if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
}
if( rc==SQLITE_OK ){
pCsr->zRank = zRank;
pCsr->zRankArgs = zRankArgs;
@ -1207,16 +1213,16 @@ static int fts5UpdateMethod(
** 3. Values for each of the nCol matchable columns.
** 4. Values for the two hidden columns (<tablename> and "rank").
*/
assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
eType0 = sqlite3_value_type(apVal[0]);
eConflict = sqlite3_vtab_on_conflict(pConfig->db);
assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
assert( pVtab->zErrMsg==0 );
assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) );
fts5TripCursors(pTab);
if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
if( eType0==SQLITE_INTEGER ){
if( fts5IsContentless(pTab) ){
pTab->base.zErrMsg = sqlite3_mprintf(
"cannot %s contentless fts5 table: %s",
@ -1227,7 +1233,8 @@ static int fts5UpdateMethod(
i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
}
}else if( nArg>1 ){
}else{
assert( nArg>1 );
sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
const char *z = (const char*)sqlite3_value_text(pCmd);
@ -1471,6 +1478,7 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
i64 iRowid = fts5CursorRowid(pCsr);
rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
}
if( iCol<0 ){
int i;
@ -1879,6 +1887,23 @@ static int fts5CreateTokenizer(
return rc;
}
static Fts5TokenizerModule *fts5LocateTokenizer(
Fts5Global *pGlobal,
const char *zName
){
Fts5TokenizerModule *pMod = 0;
if( zName==0 ){
pMod = pGlobal->pDfltTok;
}else{
for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
if( sqlite3_stricmp(zName, pMod->zName)==0 ) break;
}
}
return pMod;
}
/*
** Find a tokenizer. This is the implementation of the
** fts5_api.xFindTokenizer() method.
@ -1889,21 +1914,13 @@ static int fts5FindTokenizer(
void **ppUserData,
fts5_tokenizer *pTokenizer /* Populate this object */
){
Fts5Global *pGlobal = (Fts5Global*)pApi;
int rc = SQLITE_OK;
Fts5TokenizerModule *pTok;
Fts5TokenizerModule *pMod;
if( zName==0 ){
pTok = pGlobal->pDfltTok;
}else{
for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
}
}
if( pTok ){
*pTokenizer = pTok->x;
*ppUserData = pTok->pUserData;
pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
if( pMod ){
*pTokenizer = pMod->x;
*ppUserData = pMod->pUserData;
}else{
memset(pTokenizer, 0, sizeof(fts5_tokenizer));
rc = SQLITE_ERROR;
@ -1917,24 +1934,23 @@ int sqlite3Fts5GetTokenizer(
const char **azArg,
int nArg,
Fts5Tokenizer **ppTok,
fts5_tokenizer **ppTokApi
fts5_tokenizer **ppTokApi,
char **pzErr
){
Fts5TokenizerModule *pMod = 0;
Fts5TokenizerModule *pMod;
int rc = SQLITE_OK;
if( nArg==0 ){
pMod = pGlobal->pDfltTok;
}else{
for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
}
}
pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);
if( pMod==0 ){
assert( nArg>0 );
rc = SQLITE_ERROR;
*pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
}else{
rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
*ppTokApi = &pMod->x;
if( rc!=SQLITE_OK && pzErr ){
*pzErr = sqlite3_mprintf("error in tokenizer constructor");
}
}
if( rc!=SQLITE_OK ){

3
ext/fts5/fts5Int.h
View File

@ -61,7 +61,8 @@ int sqlite3Fts5GetTokenizer(
const char **azArg,
int nArg,
Fts5Tokenizer**,
fts5_tokenizer**
fts5_tokenizer**,
char **pzErr
);
/*

5
ext/fts5/fts5_aux.c
View File

@ -230,11 +230,12 @@ static void fts5HighlightFunction(
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3_free(ctx.zOut);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}
}
/*
** End of highlight() implementation.

10
ext/fts5/fts5_config.c
View File

@ -330,11 +330,9 @@ static int fts5ConfigParseSpecial(
rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5GetTokenizer(pGlobal,
(const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi
(const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
pzErr
);
if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("error in tokenizer constructor");
}
}
}
}
@ -387,7 +385,7 @@ static int fts5ConfigParseSpecial(
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
return sqlite3Fts5GetTokenizer(
pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
);
}
@ -563,7 +561,7 @@ int sqlite3Fts5ConfigParse(
rc = SQLITE_ERROR;
}else{
if( bOption ){
rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr);
rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
}else{
rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
zOne = 0;

6
ext/fts5/fts5_expr.c
View File

@ -280,15 +280,15 @@ int sqlite3Fts5ExprPhraseExpr(
Fts5ExprPhrase *pCopy;
pOrig = pExpr->apExprPhrase[iPhrase];
pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc,
sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
);
pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc,
sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
);
pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc,
sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
);
for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);

7
ext/fts5/fts5_tcl.c
View File

@ -36,6 +36,9 @@ int sqlite3_fts5_may_be_corrupt = 0;
** can extract the sqlite3* pointer from an existing Tcl SQLite
** connection.
*/
extern const char *sqlite3ErrName(int);
struct SqliteDb {
sqlite3 *db;
};
@ -390,7 +393,7 @@ static int xF5tApi(
#undef CASE
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in api call", 0);
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
return TCL_ERROR;
}
@ -727,8 +730,6 @@ static int f5tTokenizerTokenize(
return rc;
}
extern const char *sqlite3ErrName(int);
/*
** sqlite3_fts5_token TEXT START END POS
*/

7
ext/fts5/fts5_tokenize.c
View File

@ -529,11 +529,16 @@ static int fts5PorterCreate(
int rc = SQLITE_OK;
PorterTokenizer *pRet;
void *pUserdata = 0;
const char *zBase = "unicode61";
if( nArg>0 ){
zBase = azArg[0];
}
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer(pApi, "unicode61", &pUserdata, &pRet->tokenizer);
rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
}else{
rc = SQLITE_NOMEM;
}

2
ext/fts5/test/fts5aa.test
View File

@ -297,7 +297,7 @@ do_catchsql_test 12.2 {
} {1 {unknown special query: stuff}}
do_test 12.3 {
set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
string is integer $res
} {1}

8
ext/fts5/test/fts5al.test
View File

@ -268,6 +268,14 @@ do_execsql_test 4.3.3 {
{a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2
}
do_catchsql_test 4.4.3 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)'
} {1 {no such function: xyz}}
do_catchsql_test 4.4.4 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}
finish_test

57
ext/fts5/test/fts5aux.test Normal file
View File

@ -0,0 +1,57 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the auxiliary function APIs.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aux
proc inst {cmd i} {
$cmd xInst $i
}
sqlite3_fts5_create_function db inst inst
proc colsize {cmd i} {
$cmd xColumnSize $i
}
sqlite3_fts5_create_function db colsize colsize
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE f1 USING fts5(a, b);
INSERT INTO f1 VALUES('one two', 'two one zero');
}
do_catchsql_test 1.1 {
SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_catchsql_test 1.2 {
SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 0 1}}}
do_catchsql_test 1.3 {
SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 1 0}}}
do_catchsql_test 1.4 {
SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_catchsql_test 2.1 {
SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_execsql_test 2.2 {
SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero';
} {2 3}
finish_test

18
ext/fts5/test/fts5content.test
View File

@ -215,6 +215,24 @@ do_execsql_test 4.7 {
do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}
#-------------------------------------------------------------------------
# Test that if the 'rowid' field of a 'delete' is not an integer, no
# changes are made to the FTS index.
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=);
INSERT INTO t5(rowid, a, b) VALUES(-1, 'one', 'two');
INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four');
INSERT INTO t5(rowid, a, b) VALUES( 1, 'five', 'six');
}
set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}]
do_execsql_test 5.1 {
INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four');
SELECT md5sum(id, block) FROM t5_data;
} $::checksum
finish_test

17
ext/fts5/test/fts5corrupt.test
View File

@ -70,7 +70,24 @@ do_test 2.1 {
} {}
#--------------------------------------------------------------------
# A mundane test - missing row in the %_content table.
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(x);
INSERT INTO t3 VALUES('one o');
INSERT INTO t3 VALUES('two e');
INSERT INTO t3 VALUES('three o');
INSERT INTO t3 VALUES('four e');
INSERT INTO t3 VALUES('five o');
}
do_execsql_test 3.1 {
SELECT * FROM t3 WHERE t3 MATCH 'o'
} {{one o} {three o} {five o}}
do_catchsql_test 3.1 {
DELETE FROM t3_content WHERE rowid = 3;
SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {database disk image is malformed}}
finish_test

41
ext/fts5/test/fts5doclist.test Normal file
View File

@ -0,0 +1,41 @@
# 2015 April 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on edge cases in the doclist format.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5doclist
#-------------------------------------------------------------------------
# Create a table with 1000 columns. Then add some large documents to it.
# All text is in the right most column of the table.
#
do_test 1.0 {
set cols [list]
for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" }
execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])"
} {}
db func rnddoc fts5_rnddoc
do_execsql_test 1.1 {
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii;
}
do_execsql_test 1.2 {
INSERT INTO ccc(ccc) VALUES('integrity-check');
}
finish_test

150
ext/fts5/test/fts5fault4.test
View File

@ -22,6 +22,8 @@ ifcapable !fts5 {
return
}
if 1 {
#-------------------------------------------------------------------------
# An OOM while dropping an fts5 table.
#
@ -40,6 +42,154 @@ do_faultsim_test 1 -faults oom-* -prep {
faultsim_test_result [list 0 {}]
}
#-------------------------------------------------------------------------
# An OOM within an "ORDER BY rank" query.
#
db func rnddoc fts5_rnddoc
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE xx USING fts5(x);
INSERT INTO xx VALUES ('abc ' || rnddoc(10));
INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
} {}
faultsim_save_and_close
do_faultsim_test 2 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM xx }
} -body {
execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
} -test {
faultsim_test_result [list 0 {3 2 1}]
}
#-------------------------------------------------------------------------
# An OOM while "reseeking" an FTS cursor.
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE jj USING fts5(j);
INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s');
INSERT INTO jj(rowid, j) VALUES(202, 't w t f s');
INSERT INTO jj(rowid, j) VALUES(303, 'w t f');
INSERT INTO jj(rowid, j) VALUES(404, 't');
}
faultsim_save_and_close
do_faultsim_test 3 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM jj }
} -body {
set res [list]
db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } {
lappend res $rowid
if {$rowid==303} {
execsql { DELETE FROM jj WHERE rowid=404 }
}
}
set res
} -test {
faultsim_test_result [list 0 {101 202 303}]
}
#-------------------------------------------------------------------------
# An OOM within a special "*reads" query.
#
reset_db
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}
set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
do_faultsim_test 4 -faults oom-* -body {
db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
faultsim_test_result {0 {0 {} 3}}
}
#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {
PRAGMA encoding='utf16';
CREATE VIRTUAL TABLE x2 USING fts5(x);
INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3
INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6
INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2
INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5
INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1
}
proc rowidmod {cmd mod} {
set row [$cmd xRowid]
expr {$row % $mod}
}
sqlite3_fts5_create_function db rowidmod rowidmod
do_faultsim_test 5.1 -faults oom-* -body {
db eval {
SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND
rank MATCH "rowidmod('7')" ORDER BY rank
}
} -test {
faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}}
}
proc rowidprefix {cmd prefix} {
set row [$cmd xRowid]
set {} "${row}-${prefix}"
}
sqlite3_fts5_create_function db rowidprefix rowidprefix
set str [string repeat abcdefghijklmnopqrstuvwxyz 10]
do_faultsim_test 5.2 -faults oom-* -body {
db eval "
SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND
rank MATCH 'rowidprefix(''$::str'')'
LIMIT 1
"
} -test {
faultsim_test_result "0 {10-$::str {a b c}}"
}
}
#-------------------------------------------------------------------------
# OOM errors within auxiliary functions.
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE x3 USING fts5(xxx);
INSERT INTO x3 VALUES('a b c d c b a');
}
do_faultsim_test 6.1 -faults oom-t* -body {
db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' }
} -test {
faultsim_test_result {0 {{a b *c* d *c* b a}}}
}
proc firstinst {cmd} {
foreach {p c o} [$cmd xInst 0] {}
expr $c*100 + $o
}
sqlite3_fts5_create_function db firstinst firstinst
do_faultsim_test 6.2 -faults oom-t* -body {
db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' }
} -test {
faultsim_test_result {0 2} {1 SQLITE_NOMEM}
}
finish_test

61
ext/fts5/test/fts5plan.test Normal file
View File

@ -0,0 +1,61 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on testing the planner (xBestIndex function).
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5plan
do_execsql_test 1.0 {
CREATE TABLE t1(x, y);
CREATE VIRTUAL TABLE f1 USING fts5(ff);
}
do_eqp_test 1.1 {
SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
} {
0 0 0 {SCAN TABLE t1}
0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
}
do_eqp_test 1.2 {
SELECT * FROM t1, f1 WHERE f1 > t1.x
} {
0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
0 1 0 {SCAN TABLE t1}
}
do_eqp_test 1.3 {
SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
} {
0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
0 0 0 {USE TEMP B-TREE FOR ORDER BY}
}
do_eqp_test 1.4 {
SELECT * FROM f1 ORDER BY rank
} {
0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
0 0 0 {USE TEMP B-TREE FOR ORDER BY}
}
do_eqp_test 1.5 {
SELECT * FROM f1 WHERE rank MATCH ?
} {
0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
}
finish_test

39
ext/fts5/test/fts5rank.test Normal file
View File

@ -0,0 +1,39 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on testing queries that use the "rank" column.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rank
#-------------------------------------------------------------------------
# "ORDER BY rank" + highlight() + large poslists.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE xyz USING fts5(z);
}
do_test 1.1 {
set doc [string trim [string repeat "x y " 500]]
execsql { INSERT INTO xyz VALUES($doc) }
} {}
do_execsql_test 1.2 {
SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]
do_execsql_test 1.3 {
SELECT highlight(xyz, 0, '[', ']') FROM xyz
WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]
finish_test

11
ext/fts5/test/fts5rebuild.test
View File

@ -46,5 +46,16 @@ do_execsql_test 1.7 {
INSERT INTO f1(f1) VALUES('integrity-check');
} {}
#-------------------------------------------------------------------------
# Check that 'rebuild' may not be used with a contentless table.
#
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
}
do_catchsql_test 2.2 {
INSERT INTO nc(nc) VALUES('rebuild');
} {1 {'rebuild' may not be used with a contentless fts5 table}}
finish_test

100
ext/fts5/test/fts5restart.test
View File

@ -19,6 +19,10 @@ do_execsql_test 1.0 {
CREATE VIRTUAL TABLE f1 USING fts5(ff);
}
#-------------------------------------------------------------------------
# Run the 'optimize' command. Check that it does not disturb ongoing
# full-text queries.
#
do_test 1.1 {
for {set i 1} {$i < 1000} {incr i} {
execsql { INSERT INTO f1 VALUES('a b c d e') }
@ -30,7 +34,6 @@ do_execsql_test 1.2 {
SELECT rowid FROM f1 WHERE f1 MATCH 'c';
} $lRowid
breakpoint
do_test 1.3 {
set res [list]
db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
@ -42,6 +45,101 @@ do_test 1.3 {
set res
} $lRowid
do_test 1.4.1 {
sqlite3 db2 test.db
set res [list]
db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
if {$rowid == 100} {
set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }]
}
lappend res $rowid
}
set res
} $lRowid
do_test 1.4.2 {
db2 close
set cres
} {1 {database is locked}}
#-------------------------------------------------------------------------
# Open a couple of cursors. Then close them in the same order.
#
do_test 2.1 {
set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X]
set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X]
sqlite3_step $::s1
} {SQLITE_ROW}
do_test 2.2 {
sqlite3_step $::s2
} {SQLITE_ROW}
do_test 2.1 {
sqlite3_finalize $::s1
sqlite3_finalize $::s2
} {SQLITE_OK}
#-------------------------------------------------------------------------
# Copy data between two FTS5 tables.
#
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE f2 USING fts5(gg);
INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d';
}
do_execsql_test 3.2 {
SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e'
} $lRowid
#-------------------------------------------------------------------------
# Remove the row that an FTS5 cursor is currently pointing to. And
# various other similar things. Check that this does not disturb
# ongoing scans.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE n4 USING fts5(n);
INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5');
INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4');
INSERT INTO n4(rowid, n) VALUES(300, '2 3 4');
INSERT INTO n4(rowid, n) VALUES(400, '2 3');
INSERT INTO n4(rowid, n) VALUES(500, '3');
}
do_test 4.1 {
set res [list]
db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } {
if {$rowid==300} {
execsql { DELETE FROM n4 WHERE rowid=300 }
}
lappend res $rowid
}
set res
} {100 200 300 400 500}
do_test 4.2 {
execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
set res [list]
db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
if {$rowid==300} {
execsql { DELETE FROM n4 WHERE rowid=300 }
}
lappend res $rowid
}
set res
} {500 400 300 200 100}
do_test 4.3 {
execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
set res [list]
db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
if {$rowid==300} {
execsql { DELETE FROM n4 }
}
lappend res $rowid
}
set res
} {500 400 300}
finish_test

8
ext/fts5/test/fts5tokenizer.test
View File

@ -37,6 +37,14 @@ do_execsql_test 1.4 {
DROP TABLE ft1;
}
do_catchsql_test 1.5 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch');
} {1 {no such tokenizer: nosuch}}
do_catchsql_test 1.6 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch');
} {1 {error in tokenizer constructor}}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
INSERT INTO ft1 VALUES('embedded databases');

44
manifest
View File

@ -1,5 +1,5 @@
C Fix\san\sfts5\sbug\sin\shandling\swrites\swhile\sthere\sare\sactive\scursors.
D 2015-04-28T20:24:50.023
C Improve\sfts5\stests.
D 2015-04-29T20:54:08.849
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -104,23 +104,23 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
F ext/fts5/fts5.c 45e82b584bb3fce22c82d8521bccaa2a6e9202fe
F ext/fts5/fts5.c 932284a253cc9bb32caf047879669720680be2c3
F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a
F ext/fts5/fts5Int.h f573fe6c50471f1d66682fce282da801009c54e1
F ext/fts5/fts5_aux.c fcea18b1a2a3f95a498b52aba2983557d7678a22
F ext/fts5/fts5Int.h 2e0a1a6b77e1e014b7e9b1479ca686ff79930457
F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
F ext/fts5/fts5_buffer.c 8c8cfe7f09ca2767ab53ea883f9a0af0edb6bbae
F ext/fts5/fts5_config.c f344ffa24d2add70fd5bde2b73c44846ad7a06bd
F ext/fts5/fts5_expr.c e647a2626af5c80a6325532b23eea6d1eb252a78
F ext/fts5/fts5_config.c ecbbd5163758a958106867051892e0dfecf68b5c
F ext/fts5/fts5_expr.c 663c75dfdb1bfd8809d696357d7b55f507815098
F ext/fts5/fts5_hash.c 29d8b0668727863cc1f1efa65efe4dd78635b016
F ext/fts5/fts5_index.c de588982b0237b1605d6c37afd115b34c95c3da1
F ext/fts5/fts5_storage.c ef60fc9dcc4e274f9589165e26833173c273ae18
F ext/fts5/fts5_tcl.c af1d37fa93bcabc926aa4e89500adedbbe84a520
F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
F ext/fts5/fts5_tcl.c 19ab8cfa642950648968dcf25075d6d969900524
F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc
F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
F ext/fts5/test/fts5aa.test 87b2e01084c3e2a960ae1500dd5f0729dac2166c
F ext/fts5/test/fts5aa.test e17657bd749cb3982745ec503ce22372dee7340d
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e
F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d
@ -131,30 +131,34 @@ F ext/fts5/test/fts5ah.test d74cf8b7de5b8424f732acef69fe12122a12f2bf
F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37
F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
F ext/fts5/test/fts5al.test 6a5717faaf7f1e0e866360022d284903f3a4eede
F ext/fts5/test/fts5al.test e6bddd2c11c0d1e3ae189ee51081899d2f4ea570
F ext/fts5/test/fts5aux.test 1e475d928a3d1decf74167394db20330d7beeb0f
F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
F ext/fts5/test/fts5content.test c4f5b0fe1bc7523bb6706591d05d1194a0aec452
F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727
F ext/fts5/test/fts5content.test 532e15b541254410adc7bfb51f94631cfe82de8f
F ext/fts5/test/fts5corrupt.test 138aecc75c36c3dac9259c7f57c5bc3d009255f8
F ext/fts5/test/fts5corrupt2.test 494111fd4f2dab36499cf97718eaba1f7c11e9d0
F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f
F ext/fts5/test/fts5doclist.test 635b80ac785627841a59c583bac702b55d49fdc5
F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
F ext/fts5/test/fts5fault1.test ed71717a479bef32d05f02d9c48691011d160d4d
F ext/fts5/test/fts5fault2.test 26c3d70648f691e2cc9391e14bbc11a973656383
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
F ext/fts5/test/fts5fault4.test e860e0cf7e56f2f87330023be1f1ced44128d5c8
F ext/fts5/test/fts5fault4.test aea710bbf5680ed41afb9d3313c297d429f9feac
F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
F ext/fts5/test/fts5hash.test adb7b0442cc1c77c507f07e16d11490486e75dfa
F ext/fts5/test/fts5merge.test 453a0717881aa7784885217b2040f3f275caff03
F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
F ext/fts5/test/fts5plan.test 89783f70dab89ff936ed6f21d88959b49c853a47
F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
F ext/fts5/test/fts5prefix.test 1287803c3df0e43f536196256fb9e0e6baccb4f1
F ext/fts5/test/fts5rebuild.test ee6792715c6c528cc188e7869d67c3c655889ddb
F ext/fts5/test/fts5restart.test 4934233dd2633665198d563c79f1015e3a2b9518
F ext/fts5/test/fts5rank.test f59a6b20ec8e08cb130d833dcece59cf9cd92890
F ext/fts5/test/fts5rebuild.test 77c6613aa048f38b4a12ddfacb2e6e1342e1b066
F ext/fts5/test/fts5restart.test cd58a5fb552ac10db549482698e503f82693bcd0
F ext/fts5/test/fts5rowid.test a1b2a6d76648c734c1aab11ee1a619067e8d90e6
F ext/fts5/test/fts5tokenizer.test 7a6ee24db908c09a0dc1eba634ffa17afcc05d86
F ext/fts5/test/fts5tokenizer.test bbcde2a7473dcaa9a1fc6809aa8965acb7b846ff
F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d
F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee
F ext/fts5/test/fts5unindexed.test f388605341a476b6ab622b4c267cd168f59a5944
@ -1311,7 +1315,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 8e8136f2dc08082c2984462719d9cba0f212c92a
R 65017a9d42efb9cfacfe6e269e92cdbd
P 07f70955392697556ca2951c9b6c3a5204cd5ec0
R 07c0e8a626b67779b0890a98070e21c4
U dan
Z 91e05f6f4d01e29ed38305d1f49be921
Z 5236003ea42b936c8f2a3578dc6f6593

2
manifest.uuid
View File

@ -1 +1 @@
07f70955392697556ca2951c9b6c3a5204cd5ec0
c1f07a3aa98eac87e2747527d15e5e5562221ceb