Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens.
FossilOrigin-Name: b9fb69e55bb05a8819688ee63987f2a05c33d59b
This commit is contained in:
dan
parent
3fd651928f
commit
2f77934385
@ -3572,11 +3572,16 @@ static void fts3EvalTokenCosts(
|
||||
}
|
||||
}else if( pExpr->eType!=FTSQUERY_NOT ){
|
||||
if( pExpr->eType==FTSQUERY_OR ){
|
||||
pRoot = pExpr;
|
||||
**ppOr = pExpr;
|
||||
pRoot = pExpr->pLeft;
|
||||
**ppOr = pRoot;
|
||||
(*ppOr)++;
|
||||
}
|
||||
fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
|
||||
if( pExpr->eType==FTSQUERY_OR ){
|
||||
pRoot = pExpr->pRight;
|
||||
**ppOr = pRoot;
|
||||
(*ppOr)++;
|
||||
}
|
||||
fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
|
||||
}
|
||||
}
|
||||
@ -3736,7 +3741,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
|
||||
Fts3Expr **apOr;
|
||||
aTC = (Fts3TokenAndCost *)sqlite3_malloc(
|
||||
sizeof(Fts3TokenAndCost) * nToken
|
||||
+ sizeof(Fts3Expr *) * nOr
|
||||
+ sizeof(Fts3Expr *) * nOr * 2
|
||||
);
|
||||
apOr = (Fts3Expr **)&aTC[nToken];
|
||||
|
||||
@ -4310,7 +4315,7 @@ int sqlite3Fts3EvalPhraseStats(
|
||||
int rc = SQLITE_OK;
|
||||
int iCol;
|
||||
|
||||
if( pExpr->bDeferred ){
|
||||
if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
|
||||
assert( pCsr->nDoc>0 );
|
||||
for(iCol=0; iCol<pTab->nColumn; iCol++){
|
||||
aiOut[iCol*3 + 1] = pCsr->nDoc;
|
||||
|
||||
16
manifest
16
manifest
@ -1,5 +1,5 @@
|
||||
C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations.
|
||||
D 2011-06-13T13:48:36.083
|
||||
C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens.
|
||||
D 2011-06-13T17:00:12.630
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
|
||||
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
|
||||
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a
|
||||
F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6
|
||||
F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe
|
||||
@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
|
||||
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
|
||||
F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9
|
||||
F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
|
||||
F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559
|
||||
F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c
|
||||
F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
|
||||
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
|
||||
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
|
||||
@ -466,7 +466,7 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
|
||||
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
|
||||
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
|
||||
F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35
|
||||
F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989
|
||||
F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81
|
||||
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
|
||||
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
|
||||
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
|
||||
@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
|
||||
F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00
|
||||
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
|
||||
F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d
|
||||
P 5f6b87f420f21749aa7c72e020c50aca74890086
|
||||
R ca51faeb6c7aef68bca6b7079ff7ebda
|
||||
P e4ab6cdb101bbeb804820425cf569ee7dc2397fb
|
||||
R 7fcdbf104822a42776783c2cd9e89101
|
||||
U dan
|
||||
Z 9a71ae3656c163debcbb9f929d0e7d22
|
||||
Z 0999cf6b26be67f673d9db0918d86701
|
||||
|
||||
@ -1 +1 @@
|
||||
e4ab6cdb101bbeb804820425cf569ee7dc2397fb
|
||||
b9fb69e55bb05a8819688ee63987f2a05c33d59b
|
||||
@ -93,12 +93,45 @@ proc get_single_near_results {tbl expr arrayvar nullvar} {
|
||||
set iCol [lindex [split $key ,] 1]
|
||||
set iPhrase 0
|
||||
foreach c $counts($key) {
|
||||
if {$c>0} { incr nHit($iPhrase,$iCol) 1 }
|
||||
incr nDoc($iPhrase,$iCol) $c
|
||||
if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
|
||||
incr nHit($iPhrase,$iCol) $c
|
||||
incr iPhrase
|
||||
}
|
||||
}
|
||||
|
||||
if {[info exists ::fts3_deferred] && [llength $expr]==1} {
|
||||
set phrase [lindex $expr 0]
|
||||
set rewritten [list]
|
||||
set partial 0
|
||||
foreach tok $phrase {
|
||||
if {[lsearch $::fts3_deferred $tok]>=0} {
|
||||
lappend rewritten *
|
||||
} else {
|
||||
lappend rewritten $tok
|
||||
set partial 1
|
||||
}
|
||||
}
|
||||
if {$partial==0} {
|
||||
set tblsize [db one "SELECT count(*) FROM $tbl"]
|
||||
for {set iCol 0} {$iCol < $nCol} {incr iCol} {
|
||||
set nHit(0,$iCol) $tblsize
|
||||
set nDoc(0,$iCol) $tblsize
|
||||
}
|
||||
} elseif {$rewritten != $phrase} {
|
||||
while {[lindex $rewritten end] == "*"} {
|
||||
set rewritten [lrange $rewritten 0 end-1]
|
||||
}
|
||||
while {[lindex $rewritten 0] == "*"} {
|
||||
set rewritten [lrange $rewritten 1 end]
|
||||
}
|
||||
get_single_near_results $tbl [list $rewritten] aRewrite nullentry
|
||||
foreach docid [array names hits] {
|
||||
set aMatchinfo($docid) $aRewrite($docid)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
# Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
|
||||
# contain the output of matchinfo('x') for the document.
|
||||
#
|
||||
@ -107,8 +140,8 @@ proc get_single_near_results {tbl expr arrayvar nullvar} {
|
||||
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
|
||||
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
|
||||
lappend mi [lindex $counts($docid,$iCol) $iPhrase]
|
||||
lappend mi $nDoc($iPhrase,$iCol)
|
||||
lappend mi $nHit($iPhrase,$iCol)
|
||||
lappend mi $nDoc($iPhrase,$iCol)
|
||||
}
|
||||
}
|
||||
set aMatchinfo($docid) $mi
|
||||
@ -119,7 +152,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} {
|
||||
set nullentry [list]
|
||||
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
|
||||
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
|
||||
lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol)
|
||||
lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -219,7 +252,7 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} {
|
||||
}
|
||||
}
|
||||
|
||||
proc do_near_test {tn tbl expr} {
|
||||
proc do_fts3query_test {tn tbl expr} {
|
||||
|
||||
get_near_results $tbl $expr aMatchinfo
|
||||
set match $expr
|
||||
@ -286,19 +319,20 @@ test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1}
|
||||
|
||||
#--------------------------------------------------------------------------
|
||||
# Test cases fts3auto-2.* run some simple tests using the
|
||||
# [do_near_test] proc.
|
||||
# [do_fts3query_test] proc.
|
||||
#
|
||||
foreach {tn create} {
|
||||
1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)"
|
||||
2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)"
|
||||
3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)"
|
||||
4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)"
|
||||
5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)"
|
||||
6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)"
|
||||
1 "fts4(a, b)"
|
||||
2 "fts4(a, b, order=DESC)"
|
||||
3 "fts4(a, b, order=ASC)"
|
||||
4 "fts4(a, b, prefix=1)"
|
||||
5 "fts4(a, b, order=DESC, prefix=1)"
|
||||
6 "fts4(a, b, order=ASC, prefix=1)"
|
||||
} {
|
||||
break
|
||||
do_test 2.$tn.1 {
|
||||
catchsql { DROP TABLE t1 }
|
||||
execsql $create
|
||||
execsql "CREATE VIRTUAL TABLE t1 USING $create"
|
||||
for {set i 0} {$i<32} {incr i} {
|
||||
set doc [list]
|
||||
if {$i&0x01} {lappend doc one}
|
||||
@ -309,6 +343,7 @@ foreach {tn create} {
|
||||
execsql { INSERT INTO t1 VALUES($doc, null) }
|
||||
}
|
||||
} {}
|
||||
|
||||
foreach {tn2 expr} {
|
||||
1 {one}
|
||||
2 {one NEAR/1 five}
|
||||
@ -327,7 +362,69 @@ foreach {tn create} {
|
||||
15 {"one two"}
|
||||
16 {"one two" NOT "three four"}
|
||||
} {
|
||||
do_near_test 2.$tn.2.$tn2 t1 $expr
|
||||
do_fts3query_test 2.$tn.2.$tn2 t1 $expr
|
||||
}
|
||||
}
|
||||
|
||||
#--------------------------------------------------------------------------
|
||||
# Some test cases involving deferred tokens.
|
||||
#
|
||||
proc make_token_deferrable {tbl token} {
|
||||
set nRow [db one "SELECT count(*) FROM $tbl"]
|
||||
set pgsz [db one "PRAGMA page_size"]
|
||||
|
||||
execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')"
|
||||
execsql BEGIN
|
||||
for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
|
||||
set doc [string repeat "$token " 100]
|
||||
execsql "INSERT INTO $tbl VALUES(\$doc)"
|
||||
}
|
||||
execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')"
|
||||
execsql COMMIT
|
||||
}
|
||||
|
||||
foreach {tn create} {
|
||||
1 "fts4(x)"
|
||||
2 "fts4(x, order=DESC)"
|
||||
} {
|
||||
catchsql { DROP TABLE t1 }
|
||||
execsql "CREATE VIRTUAL TABLE t1 USING $create"
|
||||
do_execsql_test 3.$tn.1 {
|
||||
INSERT INTO t1 VALUES('a b c d e f g h i j k');
|
||||
INSERT INTO t1 VALUES('b c d e f g h i j k a');
|
||||
INSERT INTO t1 VALUES('c d e f g h i j k a b');
|
||||
INSERT INTO t1 VALUES('d e f g h i j k a b c');
|
||||
INSERT INTO t1 VALUES('e f g h i j k a b c d');
|
||||
INSERT INTO t1 VALUES('f g h i j k a b c d e');
|
||||
INSERT INTO t1 VALUES('a c e g i k');
|
||||
INSERT INTO t1 VALUES('a d g j');
|
||||
INSERT INTO t1 VALUES('c a b');
|
||||
}
|
||||
|
||||
make_token_deferrable t1 c
|
||||
|
||||
set ::fts3_deferred [list]
|
||||
foreach {tn2 expr} {
|
||||
1 {a OR c}
|
||||
} {
|
||||
do_fts3query_test 3.$tn.2.$tn2 t1 $expr
|
||||
}
|
||||
|
||||
set ::fts3_deferred [list c]
|
||||
execsql {
|
||||
UPDATE t1_segments
|
||||
SET block = zeroblob(length(block))
|
||||
WHERE length(block)>10000 AND 0
|
||||
}
|
||||
foreach {tn2 expr} {
|
||||
1 {a NEAR c}
|
||||
2 {a AND c}
|
||||
3 {"a c"}
|
||||
4 {"c a"}
|
||||
5 {"a c" NEAR/1 g}
|
||||
6 {"a c" NEAR/0 g}
|
||||
} {
|
||||
do_fts3query_test 3.$tn.2.$tn2 t1 $expr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -56,14 +56,13 @@ do_execsql_test 1.2.1 {
|
||||
SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
|
||||
} {{a b c d e f a x y}}
|
||||
|
||||
breakpoint
|
||||
do_execsql_test 1.2.2 {
|
||||
SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
|
||||
FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
|
||||
} [list \
|
||||
{a b c d [e] [f] [a] x y} \
|
||||
{0 1 8 1 0 0 10 1 0 2 12 1} \
|
||||
[list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9]
|
||||
[list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9]
|
||||
]
|
||||
|
||||
do_execsql_test 1.2.3 {
|
||||
@ -72,7 +71,7 @@ do_execsql_test 1.2.3 {
|
||||
} [list \
|
||||
{[a] b c d [e] [f] [a] x y} \
|
||||
{0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \
|
||||
[list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9]
|
||||
[list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9]
|
||||
]
|
||||
|
||||
do_execsql_test 1.3.1 { DROP TABLE t1 }
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user