Add tests for fts5 synonyms implemented by adding extra terms to queries. And fixes for the same.
FossilOrigin-Name: dbcb73802b88f76be17f09f3eb83ffac542de633
This commit is contained in:
dan
parent
cb8b58caf7
commit
3faa07ee04
@ -235,6 +235,8 @@ struct Fts5PoslistReader {
|
||||
int n; /* Size of buffer at a[] in bytes */
|
||||
int i; /* Current offset in a[] */
|
||||
|
||||
u8 bFlag; /* For client use (any custom purpose) */
|
||||
|
||||
/* Output variables */
|
||||
u8 bEof; /* Set to true at EOF */
|
||||
i64 iPos; /* (iCol<<32) + iPos */
|
||||
|
||||
@ -22,6 +22,8 @@
|
||||
*/
|
||||
#define FTS5_EOF 0
|
||||
|
||||
#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
|
||||
|
||||
typedef struct Fts5ExprTerm Fts5ExprTerm;
|
||||
|
||||
/*
|
||||
@ -386,26 +388,66 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
|
||||
static int fts5ExprSynonymPoslist(
|
||||
Fts5ExprTerm *pTerm,
|
||||
i64 iRowid,
|
||||
int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */
|
||||
u8 **pa, int *pn
|
||||
){
|
||||
Fts5PoslistWriter writer = {0};
|
||||
Fts5PoslistReader aStatic[4];
|
||||
Fts5PoslistReader *aIter = aStatic;
|
||||
int nIter = 0;
|
||||
int rc = SQLITE_OK;
|
||||
Fts5ExprTerm *p;
|
||||
|
||||
assert( pTerm->pSynonym );
|
||||
for(p=pTerm; p; p=p->pSynonym){
|
||||
Fts5IndexIter *pIter = p->pIter;
|
||||
if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
|
||||
const u8 *a;
|
||||
int n;
|
||||
i64 dummy;
|
||||
int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy);
|
||||
return rc;
|
||||
rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){
|
||||
nIter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert( 0 );
|
||||
return SQLITE_ERROR;
|
||||
assert( *pbDel==0 );
|
||||
if( nIter==1 ){
|
||||
*pa = (u8*)aIter[0].a;
|
||||
*pn = aIter[0].n;
|
||||
}else{
|
||||
Fts5PoslistWriter writer = {0};
|
||||
Fts5Buffer buf = {0,0,0};
|
||||
i64 iPrev = -1;
|
||||
while( 1 ){
|
||||
int i;
|
||||
i64 iMin = FTS5_LARGEST_INT64;
|
||||
for(i=0; i<nIter; i++){
|
||||
if( aIter[i].bEof==0 ){
|
||||
if( aIter[i].iPos==iPrev ){
|
||||
if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
|
||||
}
|
||||
if( aIter[i].iPos<iMin ){
|
||||
iMin = aIter[i].iPos;
|
||||
}
|
||||
}
|
||||
}
|
||||
if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break;
|
||||
rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin);
|
||||
iPrev = iMin;
|
||||
}
|
||||
if( rc ){
|
||||
sqlite3_free(buf.p);
|
||||
}else{
|
||||
*pa = buf.p;
|
||||
*pn = buf.n;
|
||||
*pbDel = 1;
|
||||
}
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
@ -447,21 +489,24 @@ static int fts5ExprPhraseIsMatch(
|
||||
aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
|
||||
if( !aIter ) return SQLITE_NOMEM;
|
||||
}
|
||||
memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
|
||||
|
||||
/* Initialize a term iterator for each term in the phrase */
|
||||
for(i=0; i<pPhrase->nTerm; i++){
|
||||
Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
|
||||
i64 dummy;
|
||||
int n;
|
||||
const u8 *a;
|
||||
int n = 0;
|
||||
int bFlag = 0;
|
||||
const u8 *a = 0;
|
||||
if( pTerm->pSynonym ){
|
||||
rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n);
|
||||
rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n);
|
||||
}else{
|
||||
rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy);
|
||||
}
|
||||
if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){
|
||||
goto ismatch_out;
|
||||
}
|
||||
if( rc!=SQLITE_OK ) goto ismatch_out;
|
||||
sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]);
|
||||
aIter[i].bFlag = bFlag;
|
||||
if( aIter[i].bEof ) goto ismatch_out;
|
||||
}
|
||||
|
||||
while( 1 ){
|
||||
@ -495,6 +540,9 @@ static int fts5ExprPhraseIsMatch(
|
||||
|
||||
ismatch_out:
|
||||
*pbMatch = (pPhrase->poslist.n>0);
|
||||
for(i=0; i<pPhrase->nTerm; i++){
|
||||
if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a);
|
||||
}
|
||||
if( aIter!=aStatic ) sqlite3_free(aIter);
|
||||
return rc;
|
||||
}
|
||||
@ -672,13 +720,25 @@ static int fts5ExprNearAdvanceFirst(
|
||||
/* Find the firstest rowid any synonym points to. */
|
||||
i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc);
|
||||
|
||||
/* Advance each iterator that currently points to iRowid */
|
||||
/* Advance each iterator that currently points to iRowid. Or, if iFrom
|
||||
** is valid - each iterator that points to a rowid before iFrom. */
|
||||
for(p=pTerm; p; p=p->pSynonym){
|
||||
if( sqlite3Fts5IterEof(p->pIter)==0 ){
|
||||
bEof = 0;
|
||||
if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){
|
||||
rc = sqlite3Fts5IterNext(p->pIter);
|
||||
i64 ii = sqlite3Fts5IterRowid(p->pIter);
|
||||
if( ii==iRowid
|
||||
|| (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc)
|
||||
){
|
||||
if( bFromValid ){
|
||||
rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
|
||||
}else{
|
||||
rc = sqlite3Fts5IterNext(p->pIter);
|
||||
}
|
||||
if( rc!=SQLITE_OK ) break;
|
||||
if( sqlite3Fts5IterEof(p->pIter)==0 ){
|
||||
bEof = 0;
|
||||
}
|
||||
}else{
|
||||
bEof = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -807,7 +867,7 @@ static int fts5ExprNearTest(
|
||||
** phrase is not a match, break out of the loop early. */
|
||||
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
|
||||
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
|
||||
if( pPhrase->nTerm>1 || pNear->pColset ){
|
||||
if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
|
||||
int bMatch = 0;
|
||||
rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
|
||||
if( bMatch==0 ) break;
|
||||
@ -985,7 +1045,7 @@ static int fts5ExprNearInitAll(
|
||||
&p->pIter
|
||||
);
|
||||
assert( rc==SQLITE_OK || p->pIter==0 );
|
||||
if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){
|
||||
if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
|
||||
bEof = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,15 +23,15 @@ ifcapable !fts5 {
|
||||
|
||||
foreach S {
|
||||
{zero 0}
|
||||
{one 1}
|
||||
{two 2}
|
||||
{one 1 i}
|
||||
{two 2 ii}
|
||||
{three 3 iii}
|
||||
{four 4}
|
||||
{five 5}
|
||||
{six 6}
|
||||
{seven 7}
|
||||
{eight 8}
|
||||
{nine 9}
|
||||
{four 4 iv}
|
||||
{five 5 v}
|
||||
{six 6 vi}
|
||||
{seven 7 vii}
|
||||
{eight 8 viii}
|
||||
{nine 9 ix}
|
||||
} {
|
||||
foreach s $S {
|
||||
set o [list]
|
||||
@ -186,16 +186,23 @@ do_execsql_test 3.2.0 {
|
||||
one 1 4 three 1 2 two 1 2
|
||||
}
|
||||
do_execsql_test 3.2.1 {
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one two three';
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three';
|
||||
} {1 1}
|
||||
do_execsql_test 3.2.2 {
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one two two three';
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three';
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one';
|
||||
} {1}
|
||||
do_execsql_test 3.2.2 {
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one two three';
|
||||
} {1}
|
||||
do_execsql_test 3.2.3 {
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three';
|
||||
} {1}
|
||||
do_execsql_test 3.2.4 {
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one two two three';
|
||||
} {1}
|
||||
do_execsql_test 3.2.5 {
|
||||
SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three';
|
||||
} {}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Check that expressions with synonyms can be parsed.
|
||||
# Check that expressions with synonyms can be parsed and executed.
|
||||
#
|
||||
reset_db
|
||||
sqlite3_fts5_create_tokenizer db tcl tcl_create
|
||||
@ -212,11 +219,72 @@ proc tcl_tokenize {tflags text} {
|
||||
|
||||
foreach {tn expr res} {
|
||||
1 {abc} {"abc"}
|
||||
2 {one} {"one"|"1"}
|
||||
2 {one} {"one"|"i"|"1"}
|
||||
3 {3} {"3"|"iii"|"three"}
|
||||
4 {3*} {"3"|"iii"|"three" *}
|
||||
} {
|
||||
do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
|
||||
do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
|
||||
}
|
||||
|
||||
do_execsql_test 4.2.1 {
|
||||
CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
|
||||
INSERT INTO xx VALUES('one two');
|
||||
INSERT INTO xx VALUES('three four');
|
||||
}
|
||||
|
||||
do_execsql_test 4.2.2 {
|
||||
SELECT rowid FROM xx WHERE xx MATCH '2'
|
||||
} {1}
|
||||
|
||||
do_execsql_test 4.2.3 {
|
||||
SELECT rowid FROM xx WHERE xx MATCH '3'
|
||||
} {2}
|
||||
|
||||
do_test 5.0 {
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
|
||||
}
|
||||
foreach {rowid a b} {
|
||||
1 {four v 4 i three} {1 3 five five 4 one}
|
||||
2 {5 1 3 4 i} {2 2 v two 4}
|
||||
3 {5 i 5 2 four 4 1} {iii ii five two 1}
|
||||
4 {ii four 4 one 5 three five} {one 5 1 iii 4 3}
|
||||
5 {three i v i four 4 1} {ii five five five iii}
|
||||
6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv}
|
||||
7 {ii ii two three 2 5} {iii i ii iii iii one one}
|
||||
8 {2 ii i two 3 three 2} {two iv v iii 3 five}
|
||||
9 {i 2 iv 3 five four v} {iii 4 three i three ii 1}
|
||||
} {
|
||||
execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) }
|
||||
}
|
||||
} {}
|
||||
|
||||
foreach {tn q res} {
|
||||
1 {one} {
|
||||
1 {four v 4 [i] three} {[1] 3 five five 4 [one]}
|
||||
2 {5 [1] 3 4 [i]} {2 2 v two 4}
|
||||
3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]}
|
||||
4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3}
|
||||
5 {three [i] v [i] four 4 [1]} {ii five five five iii}
|
||||
6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv}
|
||||
7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]}
|
||||
8 {2 ii [i] two 3 three 2} {two iv v iii 3 five}
|
||||
9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]}
|
||||
}
|
||||
2 {five four} {
|
||||
1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one}
|
||||
2 {[5] 1 3 [4] i} {2 2 [v] two [4]}
|
||||
3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1}
|
||||
4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3}
|
||||
5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii}
|
||||
8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]}
|
||||
9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1}
|
||||
}
|
||||
} {
|
||||
do_execsql_test 5.1.$tn {
|
||||
SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
|
||||
FROM t1 WHERE t1 MATCH $q
|
||||
} $res
|
||||
}
|
||||
|
||||
finish_test
|
||||
|
||||
16
manifest
16
manifest
@ -1,5 +1,5 @@
|
||||
C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex).
|
||||
D 2015-08-31T20:06:06.235
|
||||
C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame.
|
||||
D 2015-09-01T18:08:36.324
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in e2218eb228374422969de7b1680eda6864affcef
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
|
||||
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
|
||||
F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03
|
||||
F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42
|
||||
F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c
|
||||
F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9
|
||||
F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
|
||||
F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
|
||||
F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384
|
||||
F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c
|
||||
F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680
|
||||
F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
|
||||
F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374
|
||||
F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42
|
||||
@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
|
||||
F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
|
||||
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
|
||||
F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
|
||||
F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d
|
||||
F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde
|
||||
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
|
||||
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
|
||||
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
|
||||
@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
|
||||
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
||||
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
|
||||
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
|
||||
P 98d07d16cab92f1e7001afbe370df3ec6343fc1f
|
||||
R e2a3883c4d0440bf5b53c5a71ba46260
|
||||
P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3
|
||||
R 7ab2a9e80bd7422fd7466024fe59c669
|
||||
U dan
|
||||
Z 7c01c61f2a729597f3365ce7db7a6ce6
|
||||
Z 679d87b3074cd417fe8a5b64a65c64bd
|
||||
|
||||
@ -1 +1 @@
|
||||
ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3
|
||||
dbcb73802b88f76be17f09f3eb83ffac542de633
|
||||
Loading…
x
Reference in New Issue
Block a user