Fix bug in xInstToken() causing the wrong token to be returned.
FossilOrigin-Name: da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f
This commit is contained in:
dan
parent
43b4864a98
commit
9d373ca1c5
@ -6607,6 +6607,7 @@ struct Fts5TokenDataIter {
|
||||
Fts5TokenDataMap *aMap;
|
||||
|
||||
Fts5PoslistReader *aPoslistReader;
|
||||
int *aPoslistToIter;
|
||||
Fts5Iter *apIter[1];
|
||||
};
|
||||
|
||||
@ -6794,16 +6795,19 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
|
||||
|
||||
/* Allocate array of iterators if they are not already allocated. */
|
||||
if( pT->aPoslistReader==0 ){
|
||||
pT->aPoslistReader = sqlite3Fts5MallocZero(
|
||||
&pIter->pIndex->rc, sizeof(Fts5PoslistReader) * pT->nIter
|
||||
int nByte = pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int));
|
||||
pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero(
|
||||
&pIter->pIndex->rc, nByte
|
||||
);
|
||||
if( pT->aPoslistReader==0 ) return;
|
||||
pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter];
|
||||
}
|
||||
|
||||
/* Populate an iterator for each poslist that will be merged */
|
||||
for(ii=0; ii<pT->nIter; ii++){
|
||||
Fts5Iter *p = pT->apIter[ii];
|
||||
if( iRowid==p->base.iRowid ){
|
||||
pT->aPoslistToIter[nReader] = ii;
|
||||
sqlite3Fts5PoslistReaderInit(
|
||||
p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++]
|
||||
);
|
||||
@ -6855,7 +6859,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
|
||||
|
||||
if( eDetail==FTS5_DETAIL_FULL ){
|
||||
pT->aMap[pT->nMap].iPos = iMinPos;
|
||||
pT->aMap[pT->nMap].iIter = iMin;
|
||||
pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin];
|
||||
pT->aMap[pT->nMap].iRowid = iRowid;
|
||||
pT->nMap++;
|
||||
}
|
||||
|
||||
@ -65,10 +65,10 @@ proc term {} {
|
||||
select_one $::expanded_vocab
|
||||
}
|
||||
|
||||
# Return a document - between 3 and 6 terms.
|
||||
# Return a document - between 3 and 10 terms.
|
||||
#
|
||||
proc document {} {
|
||||
set nTerm [expr [random 3] + 3]
|
||||
set nTerm [expr [random 3] + 7]
|
||||
set doc ""
|
||||
for {set ii 0} {$ii < $nTerm} {incr ii} {
|
||||
lappend doc [term]
|
||||
@ -82,11 +82,39 @@ db func document document
|
||||
set NDOC 200
|
||||
set NLOOP 100
|
||||
|
||||
|
||||
sqlite3_fts5_register_origintext db
|
||||
|
||||
proc tokens {cmd} {
|
||||
for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
|
||||
set txt [$cmd xInstToken $iTok 0]
|
||||
if {$txt==""} break
|
||||
set txt [string map [list "\0" "."] $txt]
|
||||
lappend ret $txt
|
||||
}
|
||||
set ret
|
||||
}
|
||||
sqlite3_fts5_create_function db tokens tokens
|
||||
|
||||
proc ctrl_tokens {doc term} {
|
||||
set ret [list]
|
||||
set term [string tolower $term]
|
||||
foreach a $doc {
|
||||
if {[string tolower $a]==$term} {
|
||||
if {$a==$term} {
|
||||
lappend ret $a
|
||||
} else {
|
||||
lappend ret [string tolower $a].$a
|
||||
}
|
||||
}
|
||||
}
|
||||
set ret
|
||||
}
|
||||
db func ctrl_tokens ctrl_tokens
|
||||
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE ft USING fts5(
|
||||
x, tokenize="origintext unicode61", contentless_delete=1, content=,
|
||||
x, tokenize="origintext unicode61",
|
||||
tokendata=1
|
||||
);
|
||||
|
||||
@ -105,13 +133,20 @@ do_test 1.1 {
|
||||
|
||||
proc do_all_vocab_test {tn} {
|
||||
foreach ::v [vocab] {
|
||||
set answer [execsql {SELECT id FROM ctrl WHERE x LIKE '%' || $::v || '%'}]
|
||||
set answer [execsql {
|
||||
SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%'
|
||||
}]
|
||||
do_execsql_test $tn.$::v {
|
||||
SELECT rowid FROM ft($::v)
|
||||
SELECT rowid, tokens(ft) FROM ft($::v)
|
||||
} $answer
|
||||
}
|
||||
}
|
||||
|
||||
#execsql_pp { SELECT * FROM ctrl }
|
||||
#execsql_pp { SELECT * FROM ft }
|
||||
#fts5_aux_test_functions db
|
||||
#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
|
||||
|
||||
do_all_vocab_test 1.2
|
||||
|
||||
for {set ii 0} {$ii < $NLOOP} {incr ii} {
|
||||
|
||||
14
manifest
14
manifest
@ -1,5 +1,5 @@
|
||||
C Add\stests\sfor\susing\stokendata=1\sand\scontentless_delete=1\stogether.
|
||||
D 2023-12-04T15:08:21.125
|
||||
C Fix\sbug\sin\sxInstToken()\scausing\sthe\swrong\stoken\sto\sbe\sreturned.
|
||||
D 2023-12-04T17:05:37.721
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||
@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
|
||||
F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
|
||||
F ext/fts5/fts5_expr.c 5619c3fab45a78eb5ed3021e3b40ec3b435ef3669293e8700354aa8dd3e6c796
|
||||
F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
|
||||
F ext/fts5/fts5_index.c b31bf4f0fb51a15cc1aa54c2f337197740f4f8898347266781ca6970ca751302
|
||||
F ext/fts5/fts5_index.c 072666814be04485445c07e0e75362d13245cef6c66e07aa2060c532190d0c10
|
||||
F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0
|
||||
F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
|
||||
F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
|
||||
@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0
|
||||
F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
|
||||
F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
|
||||
F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
|
||||
F ext/fts5/test/fts5origintext5.test 5b9fa1b7d2f8c5f933076000c30aea5b104c00c3f1b767334b87b76d46492e59
|
||||
F ext/fts5/test/fts5origintext5.test f4377b67debb10e3731030ce51245b9843ffb31f85725615b3b8820bd5912702
|
||||
F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
|
||||
F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
|
||||
F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
|
||||
@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0
|
||||
R 977c39d056cdc1226d7f3117e124e785
|
||||
P a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10
|
||||
R 135aeb3f5184e1f0131c446ddc8eab37
|
||||
U dan
|
||||
Z 8168409535c63203870031d6c1a58cc5
|
||||
Z 33215e49f8892bbcd31004428ffced69
|
||||
# Remove this line to create a well-formed Fossil manifest.
|
||||
|
||||
@ -1 +1 @@
|
||||
a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10
|
||||
da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f
|
||||
Loading…
Reference in New Issue
Block a user