Add a fix and tests for the FTS deferred token logic.
FossilOrigin-Name: 91daea7d2ec41f014fb30c6371aae447cc07f287
This commit is contained in:
dan
parent
b8ae1afba1
commit
da7b45615d
@ -3864,12 +3864,14 @@ static int fts3EvalSelectDeferred(
|
||||
){
|
||||
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
|
||||
int nDocSize = 0; /* Number of pages per doc loaded */
|
||||
int nDocEst = 0; /* Est. docs if all other tokens deferred */
|
||||
int rc = SQLITE_OK; /* Return code */
|
||||
int ii; /* Iterator variable for various purposes */
|
||||
int nOvfl = 0; /* Total overflow pages used by doclists */
|
||||
int nToken = 0; /* Total number of tokens in cluster */
|
||||
|
||||
int nMinEst = 0; /* The minimum count for any phrase so far. */
|
||||
int nLoad4 = 1; /* (Phrases that will be loaded)^4. */
|
||||
|
||||
/* Count the tokens in this AND/NEAR cluster. If none of the doclists
|
||||
** associated with the tokens spill onto overflow pages, or if there is
|
||||
** only 1 token, exit early. No tokens to defer in this case. */
|
||||
@ -3885,6 +3887,29 @@ static int fts3EvalSelectDeferred(
|
||||
rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
|
||||
assert( rc!=SQLITE_OK || nDocSize>0 );
|
||||
|
||||
|
||||
/* Iterate through all tokens in this AND/NEAR cluster, in ascending order
|
||||
** of the number of overflow pages that will be loaded by the pager layer
|
||||
** to retrieve the entire doclist for the token from the full-text index.
|
||||
** Load the doclists for tokens that are either:
|
||||
**
|
||||
** a. The cheapest token in the entire query (i.e. the one visited by the
|
||||
** first iteration of this loop), or
|
||||
**
|
||||
** b. Part of a multi-token phrase.
|
||||
**
|
||||
** After each token doclist is loaded, merge it with the others from the
|
||||
** same phrase and count the number of documents that the merged doclist
|
||||
** contains. Set variable "nMinEst" to the smallest number of documents in
|
||||
** any phrase doclist for which 1 or more token doclists have been loaded.
|
||||
** Let nOther be the number of other phrases for which it is certain that
|
||||
** one or more tokens will not be deferred.
|
||||
**
|
||||
** Then, for each token, defer it if loading the doclist would result in
|
||||
** loading N or more overflow pages into memory, where N is computed as:
|
||||
**
|
||||
** (nMinEst + 4^nOther - 1) / (4^nOther)
|
||||
*/
|
||||
for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
|
||||
int iTC; /* Used to iterate through aTC[] array. */
|
||||
Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */
|
||||
@ -3899,14 +3924,7 @@ static int fts3EvalSelectDeferred(
|
||||
}
|
||||
assert( pTC );
|
||||
|
||||
/* Determine if token pTC should be deferred. If not, update nDocEst.
|
||||
**
|
||||
** TODO: If there are performance regressions involving deferred tokens,
|
||||
** this (the logic that selects the tokens to be deferred) is probably
|
||||
** the bit that needs to change.
|
||||
*/
|
||||
|
||||
if( ii && pTC->nOvfl>=(nDocEst*nDocSize) ){
|
||||
if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
|
||||
/* The number of overflow pages to load for this (and therefore all
|
||||
** subsequent) tokens is greater than the estimated number of pages
|
||||
** that will be loaded if all subsequent tokens are deferred.
|
||||
@ -3915,26 +3933,26 @@ static int fts3EvalSelectDeferred(
|
||||
rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
|
||||
fts3SegReaderCursorFree(pToken->pSegcsr);
|
||||
pToken->pSegcsr = 0;
|
||||
}else if( ii==0 || pTC->pPhrase->nToken>1 ){
|
||||
/* Either this is the cheapest token in the entire query, or it is
|
||||
** part of a multi-token phrase. Either way, the entire doclist will
|
||||
** (eventually) be loaded into memory. It may as well be now. */
|
||||
Fts3PhraseToken *pToken = pTC->pToken;
|
||||
int nList = 0;
|
||||
char *pList = 0;
|
||||
rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
|
||||
assert( rc==SQLITE_OK || pList==0 );
|
||||
if( rc==SQLITE_OK ){
|
||||
fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
|
||||
nDocEst = fts3DoclistCountDocids(
|
||||
pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
|
||||
);
|
||||
}else{
|
||||
nLoad4 = nLoad4*4;
|
||||
if( ii==0 || pTC->pPhrase->nToken>1 ){
|
||||
/* Either this is the cheapest token in the entire query, or it is
|
||||
** part of a multi-token phrase. Either way, the entire doclist will
|
||||
** (eventually) be loaded into memory. It may as well be now. */
|
||||
Fts3PhraseToken *pToken = pTC->pToken;
|
||||
int nList = 0;
|
||||
char *pList = 0;
|
||||
rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
|
||||
assert( rc==SQLITE_OK || pList==0 );
|
||||
if( rc==SQLITE_OK ){
|
||||
int nCount;
|
||||
fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
|
||||
nCount = fts3DoclistCountDocids(
|
||||
pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
|
||||
);
|
||||
if( ii==0 || nCount<nMinEst ) nMinEst = nCount;
|
||||
}
|
||||
}
|
||||
}else {
|
||||
/* This token will not be deferred. And it will not be loaded into
|
||||
** memory at this point either. So assume that it filters out 75% of
|
||||
** the currently estimated number of documents. */
|
||||
nDocEst = 1 + (nDocEst/4);
|
||||
}
|
||||
pTC->pToken = 0;
|
||||
}
|
||||
|
||||
14
manifest
14
manifest
@ -1,5 +1,5 @@
|
||||
C Merge\slatest\strunk\schanges\swith\sfts3-changes\sbranch.
|
||||
D 2011-06-28T09:51:22.803
|
||||
C Add\sa\sfix\sand\stests\sfor\sthe\sFTS\sdeferred\stoken\slogic.
|
||||
D 2011-06-28T11:58:09.194
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
|
||||
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
|
||||
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c 70127bf57c54c45545a4ddd73161caf2d9bd1abe
|
||||
F ext/fts3/fts3.c f45ad45053a587ad1c005459b704b7ade8bd504e
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3Int.h e670e2c58b3dca76e44f7e7d35e61113d079db58
|
||||
F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691
|
||||
@ -456,7 +456,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
|
||||
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
|
||||
F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9
|
||||
F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
|
||||
F test/fts3auto.test f1cb0a55130897013ca5850dbee2945c2908a45a
|
||||
F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494
|
||||
F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
|
||||
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
|
||||
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
|
||||
@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00
|
||||
F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576
|
||||
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
|
||||
F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262
|
||||
P 11ea98e454ad35b2606e81dec9ac866e4dd9f7e1 29e69f389cb0078e125ba5814a68b3fe8cf634c6
|
||||
R 99f54668f5b612c34e36e4280019aae3
|
||||
P 226686475c28cdc87e86d2d3f10014436a9a2b8f
|
||||
R 26c661c69e309ad434e75bfd241a096d
|
||||
U dan
|
||||
Z c4e55cbac13868c8cb421abab69674f2
|
||||
Z 1b1eba0343676ae88643e1d9f2f985a9
|
||||
|
||||
@ -1 +1 @@
|
||||
226686475c28cdc87e86d2d3f10014436a9a2b8f
|
||||
91daea7d2ec41f014fb30c6371aae447cc07f287
|
||||
@ -107,15 +107,17 @@ proc do_fts3query_test {tn args} {
|
||||
" $matchinfo_asc
|
||||
}
|
||||
|
||||
# fts3_make_deferrable TABLE TOKEN
|
||||
# fts3_make_deferrable TABLE TOKEN ?NROW?
|
||||
#
|
||||
proc fts3_make_deferrable {tbl token} {
|
||||
proc fts3_make_deferrable {tbl token {nRow 0}} {
|
||||
|
||||
set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy]
|
||||
set name [sqlite3_column_name $stmt 0]
|
||||
sqlite3_finalize $stmt
|
||||
|
||||
set nRow [db one "SELECT count(*) FROM $tbl"]
|
||||
if {$nRow==0} {
|
||||
set nRow [db one "SELECT count(*) FROM $tbl"]
|
||||
}
|
||||
set pgsz [db one "PRAGMA page_size"]
|
||||
execsql BEGIN
|
||||
for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
|
||||
@ -653,6 +655,50 @@ foreach {tn pending create} {
|
||||
catchsql { COMMIT }
|
||||
}
|
||||
|
||||
foreach {tn create} {
|
||||
1 "fts4(x)"
|
||||
2 "fts4(x, order=DESC)"
|
||||
} {
|
||||
execsql [subst {
|
||||
DROP TABLE IF EXISTS t1;
|
||||
CREATE VIRTUAL TABLE t1 USING $create;
|
||||
}]
|
||||
|
||||
foreach {x} {
|
||||
"F E N O T K X V A X I E X A P G Q V H U"
|
||||
"R V A E T C V Q N I E L O N U G J K L U"
|
||||
"U Y I G W M V F J L X I D C H F P J Q B"
|
||||
"S G D Z X R P G S S Y B K A S G A I L L"
|
||||
"L S I C H T Z S R Q P R N K J X L F M J"
|
||||
"C C C D P X B Z C M A D A C X S B T X V"
|
||||
"W Y J M D R G V R K B X S A W R I T N C"
|
||||
"P K L W T M S P O Y Y V V O E H Q A I R"
|
||||
"C D Y I C Z F H J C O Y A Q F L S B D K"
|
||||
"P G S C Y C Y V I M B D S Z D D Y W I E"
|
||||
"Z K Z U E E S F Y X T U A L W O U J C Q"
|
||||
"P A T Z S W L P L Q V Y Y I P W U X S S"
|
||||
"I U I H U O F Z F R H R F T N D X A G M"
|
||||
"N A B M S H K X S O Y D T X S B R Y H Z"
|
||||
"L U D A S K I L S V Z J P U B E B Y H M"
|
||||
} {
|
||||
execsql { INSERT INTO t1 VALUES($x) }
|
||||
}
|
||||
|
||||
# Add extra documents to the database such that token "B" will be considered
|
||||
# deferrable if considering the other tokens means that 2 or fewer documents
|
||||
# will be loaded into memory.
|
||||
#
|
||||
fts3_make_deferrable t1 B 2
|
||||
|
||||
# B is not deferred in either of the first two tests below, since filtering
|
||||
# on "M" or "D" returns 10 documents or so. But filtering on "M * D" only
|
||||
# returns 2, so B is deferred in this case.
|
||||
#
|
||||
do_fts3query_test 7.$tn.1 t1 {"M B"}
|
||||
do_fts3query_test 7.$tn.2 t1 {"B D"}
|
||||
do_fts3query_test 7.$tn.3 -deferred B t1 {"M B D"}
|
||||
}
|
||||
|
||||
set sqlite_fts3_enable_parentheses $sfep
|
||||
finish_test
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user