mirrors
/
sqlite
mirror of https://github.com/sqlite/sqlite
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve the performance of fts3/4 queries that use the OR operator and at least one auxiliary fts function. 

FossilOrigin-Name: 245e8730451fbdc1c729beff7295c452df604009
This commit is contained in:
dan 2015-01-27 18:43:02 +00:00
parent a58d4a9612
commit 18f6ff9eb7
5 changed files with 96 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
ext/fts3/fts3.c
View File

@ -5020,6 +5020,22 @@ static void fts3EvalNextRow(
}
pExpr->iDocid = pLeft->iDocid;
pExpr->bEof = (pLeft->bEof || pRight->bEof);
if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
Fts3Doclist *pDl = &pRight->pPhrase->doclist;
while( *pRc==SQLITE_OK && pRight->bEof==0 ){
memset(pDl->pList, 0, pDl->nList);
fts3EvalNextRow(pCsr, pRight, pRc);
}
}
if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){
Fts3Doclist *pDl = &pLeft->pPhrase->doclist;
while( *pRc==SQLITE_OK && pLeft->bEof==0 ){
memset(pDl->pList, 0, pDl->nList);
fts3EvalNextRow(pCsr, pLeft, pRc);
}
}
}
}
break;
}
@ -5392,6 +5408,7 @@ static void fts3EvalRestart(
}
pPhrase->doclist.pNextDocid = 0;
pPhrase->doclist.iDocid = 0;
pPhrase->pOrPoslist = 0;
}
pExpr->iDocid = 0;
@ -5637,6 +5654,7 @@ int sqlite3Fts3EvalPhrasePoslist(
iDocid = pExpr->iDocid;
pIter = pPhrase->doclist.pList;
if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
int rc = SQLITE_OK;
int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
int iMul; /* +1 if csr dir matches index dir, else -1 */
int bOr = 0;
@ -5662,72 +5680,43 @@ int sqlite3Fts3EvalPhrasePoslist(
** an incremental phrase. Load the entire doclist for the phrase
** into memory in this case. */
if( pPhrase->bIncr ){
int rc = SQLITE_OK;
int bEofSave = pExpr->bEof;
fts3EvalRestart(pCsr, pExpr, &rc);
while( rc==SQLITE_OK && !pExpr->bEof ){
fts3EvalNextRow(pCsr, pExpr, &rc);
if( bEofSave==0 && pExpr->iDocid==iDocid ) break;
int bEofSave = pNear->bEof;
fts3EvalRestart(pCsr, pNear, &rc);
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
if( bEofSave==0 && pNear->iDocid==iDocid ) break;
}
pIter = pPhrase->doclist.pList;
assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
if( rc!=SQLITE_OK ) return rc;
}
iMul = ((pCsr->bDesc==bDescDoclist) ? 1 : -1);
while( bTreeEof==1
&& pNear->bEof==0
&& (DOCID_CMP(pNear->iDocid, pCsr->iPrevId) * iMul)<0
){
int rc = SQLITE_OK;
fts3EvalNextRow(pCsr, pExpr, &rc);
if( rc!=SQLITE_OK ) return rc;
iDocid = pExpr->iDocid;
pIter = pPhrase->doclist.pList;
}
bEof = (pPhrase->doclist.nAll==0);
assert( bDescDoclist==0 || bDescDoclist==1 );
assert( pCsr->bDesc==0 || pCsr->bDesc==1 );
if( bEof==0 ){
if( pCsr->bDesc==bDescDoclist ){
int dummy;
if( pNear->bEof ){
/* This expression is already at EOF. So position it to point to the
** last entry in the doclist at pPhrase->doclist.aAll[]. Variable
** iDocid is already set for this entry, so all that is required is
** to set pIter to point to the first byte of the last position-list
** in the doclist.
**
** It would also be correct to set pIter and iDocid to zero. In
** this case, the first call to sqltie3Fts4DoclistPrev() below
** would also move the iterator to point to the last entry in the
** doclist. However, this is expensive, as to do so it has to
** iterate through the entire doclist from start to finish (since
** it does not know the docid for the last entry). */
pIter = &pPhrase->doclist.aAll[pPhrase->doclist.nAll-1];
fts3ReversePoslist(pPhrase->doclist.aAll, &pIter);
}
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
sqlite3Fts3DoclistPrev(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}else{
if( pNear->bEof ){
pIter = 0;
iDocid = 0;
}
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
if( bTreeEof ){
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
}
}
if( rc!=SQLITE_OK ) return rc;
pIter = pPhrase->pOrPoslist;
iDocid = pPhrase->iOrDocid;
if( pCsr->bDesc==bDescDoclist ){
bEof = (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
}else{
bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
int dummy;
sqlite3Fts3DoclistPrev(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}
pPhrase->pOrPoslist = pIter;
pPhrase->iOrDocid = iDocid;
if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
}
@ -5741,10 +5730,13 @@ int sqlite3Fts3EvalPhrasePoslist(
}
while( iThis<iCol ){
fts3ColumnlistCopy(0, &pIter);
if( *pIter==0x00 ) return 0;
if( *pIter==0x00 ) return SQLITE_OK;
pIter++;
pIter += fts3GetVarint32(pIter, &iThis);
}
if( *pIter==0x00 ){
pIter = 0;
}
*ppOut = ((iCol==iThis)?pIter:0);
return SQLITE_OK;

5
ext/fts3/fts3Int.h
View File

@ -375,6 +375,11 @@ struct Fts3Phrase {
int bIncr; /* True if doclist is loaded incrementally */
int iDoclistToken;
/* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an
** OR condition. */
char *pOrPoslist;
i64 iOrDocid;
/* Variables below this point are populated by fts3_expr.c when parsing
** a MATCH expression. Everything above is part of the evaluation phase.
*/

52
ext/fts3/fts3_snippet.c
View File

@ -442,37 +442,39 @@ static int fts3BestSnippet(
sIter.nSnippet = nSnippet;
sIter.nPhrase = nList;
sIter.iCurrent = -1;
(void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
if( rc==SQLITE_OK ){
/* Set the *pmSeen output variable. */
for(i=0; i<nList; i++){
if( sIter.aPhrase[i].pHead ){
*pmSeen |= (u64)1 << i;
/* Set the *pmSeen output variable. */
for(i=0; i<nList; i++){
if( sIter.aPhrase[i].pHead ){
*pmSeen |= (u64)1 << i;
}
}
}
/* Loop through all candidate snippets. Store the best snippet in
** *pFragment. Store its associated 'score' in iBestScore.
*/
pFragment->iCol = iCol;
while( !fts3SnippetNextCandidate(&sIter) ){
int iPos;
int iScore;
u64 mCover;
u64 mHighlight;
fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight);
assert( iScore>=0 );
if( iScore>iBestScore ){
pFragment->iPos = iPos;
pFragment->hlmask = mHighlight;
pFragment->covered = mCover;
iBestScore = iScore;
/* Loop through all candidate snippets. Store the best snippet in
** *pFragment. Store its associated 'score' in iBestScore.
*/
pFragment->iCol = iCol;
while( !fts3SnippetNextCandidate(&sIter) ){
int iPos;
int iScore;
u64 mCover;
u64 mHighlite;
fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite);
assert( iScore>=0 );
if( iScore>iBestScore ){
pFragment->iPos = iPos;
pFragment->hlmask = mHighlite;
pFragment->covered = mCover;
iBestScore = iScore;
}
}
}
*piScore = iBestScore;
}
sqlite3_free(sIter.aPhrase);
*piScore = iBestScore;
return SQLITE_OK;
return rc;
}

18
manifest
View File

@ -1,5 +1,5 @@
C Fix\sa\s(almost\salways\sharmless)\sread\spast\sthe\send\sof\sa\smemory\sallocation\nthat\scomes\sabout\sbecause\sthe\sExpr.pTab\sfield\sis\schecked\son\san\nEXPR_REDUCEDSIZE\sExpr\sobject\sbefore\schecking\sthe\sExpr.op\sfield\sto\nknow\sthat\sthe\sExpr.pTab\sfield\sis\smeaningless.
D 2015-01-27T13:17:05.225
C Improve\sthe\sperformance\sof\sfts3/4\squeries\sthat\suse\sthe\sOR\soperator\sand\sat\sleast\sone\sauxiliary\sfts\sfunction.
D 2015-01-27T18:43:02.971
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -78,16 +78,16 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 5c464816508e40feb3c61f1f5566551764698fc8
F ext/fts3/fts3.c 845f20440dacac6e09f5c7735205609b9a86536b
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 53d4eca1fb23eab00681fb028fb82eb5705c1e21
F ext/fts3/fts3Int.h 394858c12a17740f7a1f6bd372c4606d4425a8d1
F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365
F ext/fts3/fts3_expr.c 40123785eaa3ebd4c45c9b23407cc44ac0c49905
F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5
F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009
F ext/fts3/fts3_snippet.c 51beb5c1498176fd9caccaf1c75b55cb803a985a
F ext/fts3/fts3_snippet.c 55c126e07158b2a705f52dee2cdc57208d3e999a
F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763
F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038
F ext/fts3/fts3_tokenize_vtab.c becc661223db7898b213f9e8a23d75bac02408c9
@ -1237,7 +1237,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 1964e656b4b420e8d6a4ba12d270ed02db292b88
R 5d4aecd212970d14e41b3c7464003655
U drh
Z 469718f07e1956a0a1c83ab2938852ec
P e098de691002a78270540430b0df1e120582b53f
R 097e5fd012edfb2d64381b5476250a91
U dan
Z 72e3dfc5e16e519968e47157e86381d8

2
manifest.uuid
View File

@ -1 +1 @@
e098de691002a78270540430b0df1e120582b53f
245e8730451fbdc1c729beff7295c452df604009