mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modify fts3rnd.test to run tests for both "ORDER BY docid ASC" and "ORDER BY docid DESC" with both order=ASC and order=DESC FTS tables. Fixes for some bugs found.

Browse Source 
FossilOrigin-Name: 89f2f482e077241ac29a58eadf44a72a9c01f98c
This commit is contained in:
dan 2011-06-06 14:51:50 +00:00
parent 9aab071780
commit 5289b01524
5 changed files with 91 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
ext/fts3/fts3.c
View File

@ -1144,6 +1144,7 @@ fts3_init_out:
pTokenizer->pModule->xDestroy(pTokenizer);
}
}else{
assert( p->pSegments==0 );
*ppVTab = &p->base;
}
return rc;
@ -1245,6 +1246,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
}
}
assert( p->pSegments==0 );
return SQLITE_OK;
}
@ -1280,6 +1282,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_free(pCsr);
return SQLITE_OK;
}
@ -2564,6 +2567,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
}else{
rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
}
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
return rc;
}
@ -2701,7 +2705,7 @@ static int fts3ColumnMethod(
sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
int rc; /* Return Code */
int rc = SQLITE_OK; /* Return Code */
Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
Fts3Table *p = (Fts3Table *)pCursor->pVtab;
@ -2712,21 +2716,20 @@ static int fts3ColumnMethod(
/* This call is a request for the "docid" column. Since "docid" is an
** alias for "rowid", use the xRowid() method to obtain the value.
*/
sqlite3_int64 iRowid;
rc = fts3RowidMethod(pCursor, &iRowid);
sqlite3_result_int64(pContext, iRowid);
sqlite3_result_int64(pContext, pCsr->iPrevId);
}else if( iCol==p->nColumn ){
/* The extra column whose name is the same as the table.
** Return a blob which is a pointer to the cursor.
*/
sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
rc = SQLITE_OK;
}else{
rc = fts3CursorSeek(0, pCsr);
if( rc==SQLITE_OK ){
sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
}
}
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
return rc;
}
@ -2760,6 +2763,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
static int fts3BeginMethod(sqlite3_vtab *pVtab){
UNUSED_PARAMETER(pVtab);
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
assert( p->pSegments==0 );
assert( p->nPendingData==0 );
assert( p->inTransaction!=1 );
TESTONLY( p->inTransaction = 1 );
@ -2777,6 +2781,7 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
assert( p->nPendingData==0 );
assert( p->inTransaction!=0 );
assert( p->pSegments==0 );
TESTONLY( p->inTransaction = 0 );
TESTONLY( p->mxSavepoint = -1; );
return SQLITE_OK;
@ -3047,7 +3052,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
assert( p->inTransaction );
assert( p->mxSavepoint < iSavepoint );
TESTONLY( p->mxSavepoint = iSavepoint );
return sqlite3Fts3PendingTermsFlush(p);
return fts3SyncMethod(pVtab);
}
static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
@ -3459,6 +3464,10 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
return rc;
}
/*
** This function is used to iterate backwards (from the end to start)
** through doclists.
*/
void sqlite3Fts3DoclistPrev(
int bDescIdx, /* True if the doclist is desc */
char *aDoclist, /* Pointer to entire doclist */
@ -3471,6 +3480,7 @@ void sqlite3Fts3DoclistPrev(
char *p = *ppIter;
int iMul = (bDescIdx ? -1 : 1);
assert( nDoclist>0 );
assert( *pbEof==0 );
assert( p || *piDocid==0 );
assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
@ -3537,7 +3547,7 @@ static int fts3EvalPhraseNext(
if( rc==SQLITE_OK && !pDL->pList ){
*pbEof = 1;
}
}else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){
}else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll,
&pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
);
@ -3879,6 +3889,7 @@ static void fts3EvalNext(
int *pRc
){
if( *pRc==SQLITE_OK ){
assert( pExpr->bEof==0 );
pExpr->bStart = 1;
switch( pExpr->eType ){
@ -3901,7 +3912,7 @@ static void fts3EvalNext(
fts3EvalNext(pCsr, pRight, pRc);
while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
if( iDiff==0 ) break;
if( iDiff<0 ){
fts3EvalNext(pCsr, pLeft, pRc);
@ -3919,17 +3930,17 @@ static void fts3EvalNext(
case FTSQUERY_OR: {
Fts3Expr *pLeft = pExpr->pLeft;
Fts3Expr *pRight = pExpr->pRight;
int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
if( iCmp==0 ){
if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
fts3EvalNext(pCsr, pLeft, pRc);
}else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
fts3EvalNext(pCsr, pRight, pRc);
}else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
fts3EvalNext(pCsr, pLeft, pRc);
}else{
fts3EvalNext(pCsr, pLeft, pRc);
fts3EvalNext(pCsr, pRight, pRc);
}
@ -3972,6 +3983,12 @@ static void fts3EvalNext(
fts3EvalFreeDeferredDoclist(pPhrase);
*pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
pExpr->iDocid = pPhrase->doclist.iDocid;
#if 0
printf("token \"%.*s\" docid=%lld\n",
pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid
);
#endif
break;
}
}
@ -4123,8 +4140,9 @@ int sqlite3Fts3EvalPhraseDoclist(
assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
if( pExpr->bStart && !pExpr->bEof ){
pExpr->bStart = 0;
while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){
while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){
fts3EvalNext(pCsr, pExpr, &rc);
assert( !pExpr->bEof );
}
}
}

14
ext/fts3/fts3_write.c
View File

@ -3159,7 +3159,7 @@ int sqlite3Fts3UpdateMethod(
int rc = SQLITE_OK; /* Return Code */
int isRemove = 0; /* True for an UPDATE or DELETE */
sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
u32 *aSzIns; /* Sizes of inserted documents */
u32 *aSzIns = 0; /* Sizes of inserted documents */
u32 *aSzDel; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
int bInsertDone = 0;
@ -3174,12 +3174,16 @@ int sqlite3Fts3UpdateMethod(
&& sqlite3_value_type(apVal[0])==SQLITE_NULL
&& sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL
){
return fts3SpecialInsert(p, apVal[p->nColumn+2]);
rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
goto update_out;
}
/* Allocate space to hold the change in document sizes */
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
if( aSzIns==0 ) return SQLITE_NOMEM;
if( aSzIns==0 ){
rc = SQLITE_NOMEM;
goto update_out;
}
aSzDel = &aSzIns[p->nColumn+1];
memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
@ -3229,8 +3233,7 @@ int sqlite3Fts3UpdateMethod(
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(aSzIns);
return rc;
goto update_out;
}
/* If this is a DELETE or UPDATE operation, remove the old record. */
@ -3263,6 +3266,7 @@ int sqlite3Fts3UpdateMethod(
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
}
update_out:
sqlite3_free(aSzIns);
sqlite3Fts3SegmentsClose(p);
return rc;

16
manifest
View File

@ -1,5 +1,5 @@
C Clean\sup\sthe\scode\sfor\sprocessing\sFTS4\soptions\sa\sbit.
D 2011-06-06T06:55:38.365
C Modify\sfts3rnd.test\sto\srun\stests\sfor\sboth\s"ORDER\sBY\sdocid\sASC"\sand\s"ORDER\sBY\sdocid\sDESC"\swith\sboth\sorder=ASC\sand\sorder=DESC\sFTS\stables.\sFixes\sfor\ssome\sbugs\sfound.
D 2011-06-06T14:51:50.541
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c ea03065c8fbffc7258777fd9e1bf1c40404ce14a
F ext/fts3/fts3.c aafc143b59710e85a6215e8843aa2802f38397bc
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a
F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f
@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1
F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c d7eded6f5ee3032b41126047cc04b6720f59e6da
F ext/fts3/fts3_write.c ed525afd524d713abe7da174d56ad935dfc26008
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -475,7 +475,7 @@ F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba
F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676
F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad
F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c
F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd
F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402
@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P 650e1a79eda5a2134a1fbd305ab1f205a57c0892
R d2808d37d9a0cf28002d639dd1fa255c
P 0425138a2365d23b07d88fda2b1f458f112f389d
R bea111e886e30c118babca4cc7623bc5
U dan
Z 406d2ac578f3de4e57d0cc24bb1a90c8
Z 6f031e903f656ed90101feb3a80e60ca

2
manifest.uuid
View File

@ -1 +1 @@
0425138a2365d23b07d88fda2b1f458f112f389d
89f2f482e077241ac29a58eadf44a72a9c01f98c

58
test/fts3rnd.test
View File

@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} {
# This [proc] is used to test the FTS3 matchinfo() function.
#
proc simple_token_matchinfo {zToken} {
proc simple_token_matchinfo {zToken bDesc} {
set nDoc(0) 0
set nDoc(1) 0
@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} {
set nHit(1) 0
set nHit(2) 0
set dir -inc
if {$bDesc} { set dir -dec }
foreach key [array names ::t1] {
set value $::t1($key)
@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} {
}
set ret [list]
foreach docid [lsort -integer [array names a]] {
foreach docid [lsort -integer $dir [array names a]] {
if { [lindex [lsort -integer $a($docid)] end] } {
set matchinfo [list 1 3]
foreach i {0 1 2} hit $a($docid) {
@ -262,17 +264,30 @@ proc mit {blob} {
return $r
}
db func mit mit
set sqlite_fts3_enable_parentheses 1
foreach nodesize {50 500 1000 2000} {
proc do_orderbydocid_test {tn sql res} {
uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res]
uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \
[lsort -int -dec $res]
]
}
foreach {nodesize order} {
50 DESC
50 ASC
500 ASC
1000 DESC
2000 ASC
} {
catch { array unset ::t1 }
set testname "$nodesize/$order"
# Create the FTS3 table. Populate it (and the Tcl array) with 100 rows.
#
db transaction {
catchsql { DROP TABLE t1 }
execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)"
execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)"
execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')"
for {set i 0} {$i < 100} {incr i} { insert_row $i }
}
@ -287,6 +302,8 @@ foreach nodesize {50 500 1000 2000} {
set nRep 2
}
set ::testprefix fts3rnd-1.$testname.$iTest
# Delete one row, update one row and insert one row.
#
set rows [array names ::t1]
@ -307,7 +324,7 @@ foreach nodesize {50 500 1000 2000} {
if {0==($iTest%2)} { execsql COMMIT }
if {0==($iTest%2)} {
do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok
#do_test 0 { fts3_integrity_check t1 } ok
}
# Pick 10 terms from the vocabulary. Check that the results of querying
@ -317,9 +334,14 @@ foreach nodesize {50 500 1000 2000} {
#
for {set i 0} {$i < 10} {incr i} {
set term [random_term]
do_select_test fts3rnd-1.$nodesize.$iTest.1.$i {
do_select_test 1.$i {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
} [simple_token_matchinfo $term]
ORDER BY docid ASC
} [simple_token_matchinfo $term 0]
do_select_test 1.$i {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
ORDER BY docid DESC
} [simple_token_matchinfo $term 1]
}
# This time, use the first two characters of each term as a term prefix
@ -329,7 +351,7 @@ foreach nodesize {50 500 1000 2000} {
for {set i 0} {$i < $nRep} {incr i} {
set prefix [string range [random_term] 0 end-1]
set match "${prefix}*"
do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
do_orderbydocid_test 2.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $match]
}
@ -339,7 +361,7 @@ foreach nodesize {50 500 1000 2000} {
for {set i 0} {$i < $nRep} {incr i} {
set term [list [random_term] [random_term]]
set match "\"$term\""
do_select_test fts3rnd-1.$nodesize.$iTest.3.$i {
do_orderbydocid_test 3.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $term]
}
@ -349,7 +371,7 @@ foreach nodesize {50 500 1000 2000} {
for {set i 0} {$i < $nRep} {incr i} {
set term [list [random_term] [random_term] [random_term]]
set match "\"$term\""
do_select_test fts3rnd-1.$nodesize.$iTest.4.$i {
do_orderbydocid_test 4.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $term]
}
@ -362,17 +384,19 @@ foreach nodesize {50 500 1000 2000} {
append query "[string range [random_term] 0 end-1]*"
set match "\"$query\""
do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
do_orderbydocid_test 5.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $query]
}
# A NEAR query with terms as the arguments.
# A NEAR query with terms as the arguments:
#
# ... MATCH '$term1 NEAR $term2' ...
#
for {set i 0} {$i < $nRep} {incr i} {
set terms [list [random_term] [random_term]]
set match [join $terms " NEAR "]
do_select_test fts3rnd-1.$nodesize.$iTest.6.$i {
do_orderbydocid_test 6.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_near $terms 10]
}
@ -383,7 +407,7 @@ foreach nodesize {50 500 1000 2000} {
set terms [list [random_term] [random_term] [random_term]]
set nNear 11
set match [join $terms " NEAR/$nNear "]
do_select_test fts3rnd-1.$nodesize.$iTest.7.$i {
do_orderbydocid_test 7.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_near $terms $nNear]
}
@ -399,7 +423,7 @@ foreach nodesize {50 500 1000 2000} {
set term1 [random_term]
set term2 [random_term]
set match "$term1 $op $term2"
do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
do_orderbydocid_test $tn.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [$proc [simple_phrase $term1] [simple_phrase $term2]]
}
@ -418,7 +442,7 @@ foreach nodesize {50 500 1000 2000} {
set term3 [random_term]
set term4 [random_term]
set match "$term1 NEAR $term2 $op $term3 NEAR $term4"
do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
do_orderbydocid_test $tn.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [$proc \
[simple_near [list $term1 $term2] 10] \