Fix a problem with NEAR queries executed inside a transaction that writes the FTS table.

FossilOrigin-Name: 051c756c367837908f6691c0a36108e088c94f99
This commit is contained in:
dan 2011-06-16 16:06:05 +00:00
parent 6b96771b82
commit d4d21fea8e
6 changed files with 173 additions and 91 deletions

@ -3493,7 +3493,7 @@ static int fts3EvalPhraseNext(
while( pIter<pEnd && *pIter==0 ) pIter++;
pDL->pNextDocid = pIter;
assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] );
assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter );
*pbEof = 0;
}
}
@ -4131,13 +4131,14 @@ static void fts3EvalRestart(
if( pPhrase ){
fts3EvalZeroPoslist(pPhrase);
if( pPhrase->bIncr ){
sqlite3Fts3EvalPhraseCleanup(pPhrase);
memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
*pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0);
}else{
pPhrase->doclist.pNextDocid = 0;
pPhrase->doclist.iDocid = 0;
assert( pPhrase->nToken==1 );
assert( pPhrase->aToken[0].pSegcsr );
sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
*pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
}
pPhrase->doclist.pNextDocid = 0;
pPhrase->doclist.iDocid = 0;
}
pExpr->iDocid = 0;

@ -439,6 +439,7 @@ struct Fts3MultiSegReader {
int nBuffer; /* Allocated size of aBuffer[] in bytes */
int iColFilter; /* If >=0, filter for this column */
int bRestart;
/* Used by fts3.c only. */
int nCost; /* Cost of running iterator */

@ -1247,6 +1247,7 @@ static int fts3SegReaderNextDocid(
pReader->pOffsetList = p;
}
}else{
char *pEnd = &pReader->aDoclist[pReader->nDoclist];
/* Pointer p currently points at the first byte of an offset list. The
** following block advances it to point one byte past the end of
@ -1275,13 +1276,15 @@ static int fts3SegReaderNextDocid(
*ppOffsetList = pReader->pOffsetList;
*pnOffsetList = (int)(p - pReader->pOffsetList - 1);
}
while( p<pEnd && *p==0 ) p++;
/* If there are no more entries in the doclist, set pOffsetList to
** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
** Fts3SegReader.pOffsetList to point to the next offset list before
** returning.
*/
if( p>=&pReader->aDoclist[pReader->nDoclist] ){
if( p>=pEnd ){
pReader->pOffsetList = 0;
}else{
rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
@ -2265,51 +2268,27 @@ static void fts3ColumnFilter(
*pnList = nList;
}
int sqlite3Fts3MsrIncrStart(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr, /* Cursor object */
int iCol, /* Column to match on. */
const char *zTerm, /* Term to iterate through a doclist for */
int nTerm /* Number of bytes in zTerm */
/*
** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any
** existing data). Grow the buffer if required.
**
** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered
** trying to resize the buffer, return SQLITE_NOMEM.
*/
static int fts3MsrBufferData(
Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
char *pList,
int nList
){
int i;
int nSegment = pCsr->nSegment;
int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
);
assert( pCsr->pFilter==0 );
assert( zTerm && nTerm>0 );
/* Advance each segment iterator until it points to the term zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
do {
int rc = fts3SegReaderNext(p, pSeg, 1);
if( rc!=SQLITE_OK ) return rc;
}while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
if( nList>pMsr->nBuffer ){
char *pNew;
pMsr->nBuffer = nList*2;
pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer);
if( !pNew ) return SQLITE_NOMEM;
pMsr->aBuffer = pNew;
}
fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
/* Determine how many of the segments actually point to zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
break;
}
}
pCsr->nAdvance = i;
/* Advance each of the segments to point to the first docid. */
for(i=0; i<pCsr->nAdvance; i++){
int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
if( rc!=SQLITE_OK ) return rc;
}
fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
assert( iCol<0 || iCol<p->nColumn );
pCsr->iColFilter = iCol;
memcpy(pMsr->aBuffer, pList, nList);
return SQLITE_OK;
}
@ -2363,49 +2342,135 @@ int sqlite3Fts3MsrIncrNext(
}
if( nList>0 ){
if( fts3SegReaderIsPending(apSegment[0]) ){
rc = fts3MsrBufferData(pMsr, pList, nList+1);
if( rc!=SQLITE_OK ) return rc;
*paPoslist = pMsr->aBuffer;
assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 );
}else{
*paPoslist = pList;
}
*piDocid = iDocid;
*paPoslist = pList;
*pnPoslist = nList;
break;
}
}
}
return SQLITE_OK;
}
static int fts3SegReaderStart(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr, /* Cursor object */
const char *zTerm, /* Term searched for (or NULL) */
int nTerm /* Length of zTerm in bytes */
){
int i;
int nSeg = pCsr->nSegment;
/* If the Fts3SegFilter defines a specific term (or term prefix) to search
** for, then advance each segment iterator until it points to a term of
** equal or greater value than the specified term. This prevents many
** unnecessary merge/sort operations for the case where single segment
** b-tree leaf nodes contain more than one term.
*/
for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
do {
int rc = fts3SegReaderNext(p, pSeg, 0);
if( rc!=SQLITE_OK ) return rc;
}while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
}
fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp);
return SQLITE_OK;
}
int sqlite3Fts3SegReaderStart(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr, /* Cursor object */
Fts3SegFilter *pFilter /* Restrictions on range of iteration */
){
int i;
/* Initialize the cursor object */
pCsr->pFilter = pFilter;
return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm);
}
/* If the Fts3SegFilter defines a specific term (or term prefix) to search
** for, then advance each segment iterator until it points to a term of
** equal or greater value than the specified term. This prevents many
** unnecessary merge/sort operations for the case where single segment
** b-tree leaf nodes contain more than one term.
*/
for(i=0; i<pCsr->nSegment; i++){
int nTerm = pFilter->nTerm;
const char *zTerm = pFilter->zTerm;
int sqlite3Fts3MsrIncrStart(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr, /* Cursor object */
int iCol, /* Column to match on. */
const char *zTerm, /* Term to iterate through a doclist for */
int nTerm /* Number of bytes in zTerm */
){
int i;
int rc;
int nSegment = pCsr->nSegment;
int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
);
assert( pCsr->pFilter==0 );
assert( zTerm && nTerm>0 );
/* Advance each segment iterator until it points to the term zTerm/nTerm. */
rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm);
if( rc!=SQLITE_OK ) return rc;
/* Determine how many of the segments actually point to zTerm/nTerm. */
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = pCsr->apSegment[i];
do {
int rc = fts3SegReaderNext(p, pSeg, 0);
if( rc!=SQLITE_OK ) return rc;
}while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
break;
}
}
fts3SegReaderSort(
pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
pCsr->nAdvance = i;
/* Advance each of the segments to point to the first docid. */
for(i=0; i<pCsr->nAdvance; i++){
rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
if( rc!=SQLITE_OK ) return rc;
}
fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
assert( iCol<0 || iCol<p->nColumn );
pCsr->iColFilter = iCol;
return SQLITE_OK;
}
/*
** This function is called on a MultiSegReader that has been started using
** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also
** have been made. Calling this function puts the MultiSegReader in such
** a state that if the next two calls are:
**
** sqlite3Fts3SegReaderStart()
** sqlite3Fts3SegReaderStep()
**
** then the entire doclist for the term is available in
** MultiSegReader.aDoclist/nDoclist.
*/
int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){
int i; /* Used to iterate through segment-readers */
assert( pCsr->zTerm==0 );
assert( pCsr->nTerm==0 );
assert( pCsr->aDoclist==0 );
assert( pCsr->nDoclist==0 );
pCsr->nAdvance = 0;
pCsr->bRestart = 1;
for(i=0; i<pCsr->nSegment; i++){
pCsr->apSegment[i]->pOffsetList = 0;
pCsr->apSegment[i]->nOffsetList = 0;
pCsr->apSegment[i]->iDocid = 0;
}
return SQLITE_OK;
}
int sqlite3Fts3SegReaderStep(
Fts3Table *p, /* Virtual table handle */
Fts3MultiSegReader *pCsr /* Cursor object */
@ -2478,9 +2543,14 @@ int sqlite3Fts3SegReaderStep(
&& !isIgnoreEmpty
&& (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
){
pCsr->aDoclist = apSegment[0]->aDoclist;
pCsr->nDoclist = apSegment[0]->nDoclist;
rc = SQLITE_ROW;
if( fts3SegReaderIsPending(apSegment[0]) ){
rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist);
pCsr->aDoclist = pCsr->aBuffer;
}else{
pCsr->aDoclist = apSegment[0]->aDoclist;
}
if( rc==SQLITE_OK ) rc = SQLITE_ROW;
}else{
int nDoclist = 0; /* Size of doclist */
sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */

@ -1,5 +1,5 @@
C Changes\sto\s#ifdefs\sso\sthat\sthe\sbuild\sgoes\scorrectly\sif\sthe\sonly\sFTS\smacro\ndefined\sis\sSQLITE_ENABLE_FTS4.
D 2011-06-16T00:54:45.816
C Fix\sa\sproblem\swith\sNEAR\squeries\sexecuted\sinside\sa\stransaction\sthat\swrites\sthe\sFTS\stable.
D 2011-06-16T16:06:05.320
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 014e06f585f7ff0e34229a8eb7955ae03b8086fe
F ext/fts3/fts3.c 78b02b5f0195e397c4239ef9213e5506b7d3fa97
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 27b0b75c4d50f4022eeacbd62801d8e204ae2584
F ext/fts3/fts3Int.h df761492ae2308d3d56123907ca29cdf9bdd3748
F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691
F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b
F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb
@ -76,7 +76,7 @@ F ext/fts3/fts3_test.c 4e833729c13cea9a6bb98d3b353f6e3b8f756004
F ext/fts3/fts3_tokenizer.c 90ba6cdd8bb1b3686ab7a3d72333131e13c8fdb2
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68
F ext/fts3/fts3_write.c b0df39416b69e735c1209a2c1cd61bc3cd21c2c9
F ext/fts3/fts3_write.c 5774a7ee9632355ebf1ec4b7a5071fc9ab9eb956
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9
F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
F test/fts3auto.test a98cc895bd92df14ce4a6e94f5c68d33edcc1372
F test/fts3auto.test b0d360b331ff68bd9fb497a6192d23dc0783637c
F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d
P 03d9480fc4a8310bc8da81a64f9206c8f4b501d9
R d0766e6b55a873753b193b31dc04f4d0
U drh
Z d9decbc0f2b53d6559c9602b3b81dc73
P a0b43a320e6491de7070966ed7c3ec55fd660a85
R a91f82245adaecbc51456523e27b5133
U dan
Z 67955b067165a6636fb02e1c55fe6931

@ -1 +1 @@
a0b43a320e6491de7070966ed7c3ec55fd660a85
051c756c367837908f6691c0a36108e088c94f99

@ -521,6 +521,7 @@ foreach {tn create} {
do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five}
do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five}
do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five}
do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five}
do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
@ -534,18 +535,23 @@ foreach {tn create} {
# The following test cases - fts3auto-5.* - focus on using prefix indexes.
#
set chunkconfig [fts3_configure_incr_load 1 1]
foreach {tn create} {
1 "fts4(a, b)"
2 "fts4(a, b, order=DESC, prefix=1)"
3 "fts4(a, b, order=ASC, prefix=1,3)"
4 "fts4(a, b, order=DESC, prefix=2,4)"
foreach {tn create pending} {
2 "fts4(a, b, order=ASC, prefix=1)" 1
1 "fts4(a, b)" 1
3 "fts4(a, b, order=ASC, prefix=1,3)" 0
4 "fts4(a, b, order=DESC, prefix=2,4)" 0
5 "fts4(a, b, order=DESC, prefix=1)" 0
6 "fts4(a, b, order=ASC, prefix=1,3)" 0
} {
execsql [subst {
DROP TABLE t1;
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING $create;
}]
if {$pending} {execsql BEGIN}
foreach {a b} {
"the song of songs which is solomons"
"let him kiss me with the kisses of his mouth for thy love is better than wine"
@ -568,12 +574,16 @@ foreach {tn create} {
execsql {INSERT INTO t1(a, b) VALUES($a, $b)}
}
do_fts3query_test 5.$tn.1.1 t1 {s*}
do_fts3query_test 5.$tn.1.2 t1 {so*}
do_fts3query_test 5.$tn.1.3 t1 {"s* o*"}
do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*}
do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*}
do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"}
do_fts3query_test 5.$tn.1.5 t1 {a*}
do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*}
do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"}
if {$pending} {execsql COMMIT}
}
eval fts3_configure_incr_load $chunkconfig