mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug in fts5.

Browse Source 
FossilOrigin-Name: a4c35fa2c94fe34b376670244fe72303c99868c1
This commit is contained in:
dan 2015-09-02 17:34:22 +00:00
parent 583611df65
commit bea34fc53a
6 changed files with 151 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
ext/fts5/fts5_expr.c
View File

@ -288,8 +288,8 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
** Argument pTerm must be a synonym iterator. Return the current rowid
** that it points to.
*/
static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
i64 iRet;
static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
i64 iRet = 0;
int bRetValid = 0;
Fts5ExprTerm *p;
@ -305,7 +305,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
}
}
assert( bRetValid );
if( pbEof && bRetValid==0 ) *pbEof = 1;
return iRet;
}
@ -346,9 +346,9 @@ static int fts5ExprSynonymPoslist(
nAlloc = nAlloc*2;
aIter = aNew;
}
if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){
nIter++;
}
sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]);
assert( aIter[nIter].bEof==0 );
nIter++;
}
}
@ -659,7 +659,7 @@ static int fts5ExprNearAdvanceFirst(
Fts5ExprTerm *p;
/* Find the firstest rowid any synonym points to. */
i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc);
i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
/* Advance each iterator that currently points to iRowid. Or, if iFrom
** is valid - each iterator that points to a rowid before iFrom. */
@ -738,6 +738,35 @@ static int fts5ExprAdvanceto(
return 0;
}
static int fts5ExprSynonymAdvanceto(
Fts5ExprTerm *pTerm, /* Term iterator to advance */
int bDesc, /* True if iterator is "rowid DESC" */
i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
int *pRc, /* OUT: Error code */
int *pbEof /* OUT: Set to true if EOF */
){
int rc = SQLITE_OK;
i64 iLast = *piLast;
Fts5ExprTerm *p;
for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
if( sqlite3Fts5IterEof(p->pIter)==0 ){
i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
}
}
}
if( rc!=SQLITE_OK ){
*pbEof = 1;
}else{
*piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof);
}
return rc;
}
/*
** IN/OUT parameter (*pa) points to a position list n bytes in size. If
** the position list contains entries for column iCol, then (*pa) is set
@ -906,7 +935,7 @@ static int fts5ExprNearNextMatch(
** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
** means the minimum rowid. */
if( pLeft->aTerm[0].pSynonym ){
iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc);
iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
}else{
iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
}
@ -920,21 +949,13 @@ static int fts5ExprNearNextMatch(
if( pTerm->pSynonym ){
Fts5ExprTerm *p;
int bEof = 1;
i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc);
i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
if( iRowid==iLast ) continue;
for(p=pTerm; p; p=p->pSynonym){
Fts5IndexIter *pIter = p->pIter;
int dummy;
if( 0==sqlite3Fts5IterEof(pIter)
&& 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0
){
bEof = 0;
}
}
if( bEof || rc ){
pNode->bEof = 1;
bMatch = 0;
if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){
return rc;
}
}else{
Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
i64 iRowid = sqlite3Fts5IterRowid(pIter);

58
ext/fts5/test/fts5fault6.test
View File

@ -22,6 +22,8 @@ ifcapable !fts5 {
return
}
if 1 {
#-------------------------------------------------------------------------
# OOM while rebuilding an FTS5 table.
#
@ -148,8 +150,14 @@ do_faultsim_test 4.1 -faults oom-t* -prep {
faultsim_test_result {0 {}}
}
}
#-------------------------------------------------------------------------
# OOM while running a query that includes synonyms and matchinfo().
#
# 5.2.* OOM while running a query that includes synonyms and matchinfo().
#
# 5.3.* OOM while running a query that returns a row containing instances
# of more than 4 synonyms for a single term.
#
proc mit {blob} {
set scan(littleEndian) i*
@ -174,6 +182,7 @@ db func mit mit
sqlite3_fts5_register_matchinfo db
do_test 5.0 {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) }
execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
foreach {rowid text} {
1 {aaaa cc b aaaaa cc aa}
2 {aa aa bb a bbb}
@ -184,6 +193,37 @@ do_test 5.0 {
7 {aaaaa aa aa ccccc bb}
8 {ccc bbbbb ccccc bbb c}
9 {cccccc bbbb a aaa cccc c}
20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee}
21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e}
22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd}
23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff}
24 {eee dd ee dddd dddd eeeeee e eee fff ffff}
25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f}
26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee}
27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d}
28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd}
29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee}
30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee}
31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd}
32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff}
33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd}
34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff}
35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff}
36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff}
37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee}
38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee}
39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd}
40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f}
41 {ddd ddd fff fffff ee fffff f fff ddddd fffff}
42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee}
43 {d dd fff fffff d f fff e dddd ee ee}
44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee}
45 {eeee f eeeee ee e ffff f ddd e fff}
46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee}
47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee}
48 {ffff ffff e dddd ffffff dd dd dddd f fffff}
49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee}
} {
execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) }
}
@ -196,9 +236,12 @@ set res [list {*}{
7 {3 24 8 1 12 6}
9 {2 24 8 3 12 6}
}]
do_execsql_test 5.1 {
do_execsql_test 5.1.1 {
SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
} $res
do_execsql_test 5.1.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f'
} 29
faultsim_save_and_close
do_faultsim_test 5.2 -faults oom* -prep {
@ -214,5 +257,16 @@ do_faultsim_test 5.2 -faults oom* -prep {
faultsim_test_result [list 0 $::res]
}
do_faultsim_test 5.3 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
} -body {
db eval {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
}
} -test {
faultsim_test_result {0 29}
}
finish_test

19
ext/fts5/test/fts5synonym.test
View File

@ -93,6 +93,7 @@ foreach {tn expr res} {
2 "eight OR 8 OR 5" {2 3}
3 "10" {}
4 "1*" {1}
5 "1 + 2" {1}
} {
do_execsql_test 2.1.$tn {
SELECT rowid FROM ft WHERE ft MATCH $expr
@ -233,6 +234,7 @@ do_test 5.0 {
}
} {}
foreach {tn q res} {
1 {one} {
1 {four v 4 [i] three} {[1] 3 five five 4 [one]}
@ -265,6 +267,23 @@ foreach {tn q res} {
8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]}
9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]}
}
4 {5 + 1} {
2 {[5 1] 3 4 i} {2 2 v two 4}
3 {[5 i] 5 2 four 4 1} {iii ii five two 1}
4 {ii four 4 one 5 three five} {one [5 1] iii 4 3}
5 {three i [v i] four 4 1} {ii five five five iii}
}
5 {one + two + three} {
7 {ii ii two three 2 5} {iii [i ii iii] iii one one}
8 {2 ii [i two 3] three 2} {two iv v iii 3 five}
}
6 {"v v"} {
1 {four v 4 i three} {1 3 [five five] 4 one}
5 {three i v i four 4 1} {ii [five five five] iii}
}
} {
do_execsql_test 5.1.$tn {
SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')

46
main.mk
View File

@ -47,6 +47,7 @@
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth
TCCX += -I$(TOP)/ext/fts5
# Object files for the SQLite library.
#
@ -230,6 +231,29 @@ SRC += \
$(TOP)/ext/rbu/sqlite3rbu.h
# FTS5 things
#
FTS5_HDR = \
$(TOP)/ext/fts5/fts5.h \
$(TOP)/ext/fts5/fts5Int.h \
fts5parse.h
FTS5_SRC = \
$(TOP)/ext/fts5/fts5_aux.c \
$(TOP)/ext/fts5/fts5_buffer.c \
$(TOP)/ext/fts5/fts5_main.c \
$(TOP)/ext/fts5/fts5_config.c \
$(TOP)/ext/fts5/fts5_expr.c \
$(TOP)/ext/fts5/fts5_hash.c \
$(TOP)/ext/fts5/fts5_index.c \
fts5parse.c \
$(TOP)/ext/fts5/fts5_storage.c \
$(TOP)/ext/fts5/fts5_tokenize.c \
$(TOP)/ext/fts5/fts5_unicode2.c \
$(TOP)/ext/fts5/fts5_varint.c \
$(TOP)/ext/fts5/fts5_vocab.c \
# Generated source code files
#
SRC += \
@ -636,25 +660,6 @@ fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
# FTS5 things
#
FTS5_SRC = \
$(TOP)/ext/fts5/fts5.h \
$(TOP)/ext/fts5/fts5Int.h \
$(TOP)/ext/fts5/fts5_aux.c \
$(TOP)/ext/fts5/fts5_buffer.c \
$(TOP)/ext/fts5/fts5_main.c \
$(TOP)/ext/fts5/fts5_config.c \
$(TOP)/ext/fts5/fts5_expr.c \
$(TOP)/ext/fts5/fts5_hash.c \
$(TOP)/ext/fts5/fts5_index.c \
fts5parse.c fts5parse.h \
$(TOP)/ext/fts5/fts5_storage.c \
$(TOP)/ext/fts5/fts5_tokenize.c \
$(TOP)/ext/fts5/fts5_unicode2.c \
$(TOP)/ext/fts5/fts5_varint.c \
$(TOP)/ext/fts5/fts5_vocab.c \
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
@ -662,11 +667,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
fts5parse.h: fts5parse.c
fts5.c: $(FTS5_SRC)
fts5.c: $(FTS5_SRC) $(FTS5_HDR)
tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl
cp $(TOP)/ext/fts5/fts5.h .
userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c

18
manifest
View File

@ -1,5 +1,5 @@
C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms.
D 2015-09-02T14:17:38.670
C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5.
D 2015-09-02T17:34:22.663
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in e2218eb228374422969de7b1680eda6864affcef
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b
F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384
F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b
F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d
F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374
F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e
@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7
F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010
F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6
F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145
@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475
F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@ -259,7 +259,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F main.mk 04840e8277ab5159af16172eafd214dae7cffff5
F main.mk 8a8468564b2e387116614fcda2a6d4f5f754b955
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P cf3e45e76d23e10ee06296c3561a341591597a04
R 0e89c166acdebcc1b36d66580652787d
P 11fa980897c6c7be218bbd9c4cd8253272d2c300
R 6df2cc151a0891be96a1f7bf13aeaf04
U dan
Z 84847a00fa2f61f76712fa9a6aa07879
Z 72b6e5381768fc37d01c74eff21e6a40

2
manifest.uuid
View File

@ -1 +1 @@
11fa980897c6c7be218bbd9c4cd8253272d2c300
a4c35fa2c94fe34b376670244fe72303c99868c1