mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix problems with combining content= and languageid= in a single fts4 table.

Browse Source 
FossilOrigin-Name: 22491e7bc38aee43819b888e04241cb6a6ef73a3
This commit is contained in:
dan 2012-03-02 19:53:02 +00:00
parent c0db214b10
commit 7395599031
4 changed files with 106 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
ext/fts3/fts3.c
View File

@ -800,13 +800,18 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
}
if( p->zLanguageid ) fts3Appendf(pRc, &zRet, ",langid");
if( p->zLanguageid ){
fts3Appendf(pRc, &zRet, ", x.%Q", "langid");
}
sqlite3_free(zFree);
}else{
fts3Appendf(pRc, &zRet, "rowid");
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
}
if( p->zLanguageid ){
fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid);
}
}
fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x",
p->zDb,
@ -1215,8 +1220,20 @@ static int fts3InitVtab(
sqlite3_free((void*)aCol);
aCol = 0;
rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
/* If a languageid= option was specified, remove the language id
** column from the aCol[] array. */
if( rc==SQLITE_OK && zLanguageid ){
int j;
for(j=0; j<nCol; j++){
if( sqlite3_stricmp(zLanguageid, aCol[j])==0 ){
memmove(&aCol[j], &aCol[j+1], (nCol-j) * sizeof(aCol[0]));
nCol--;
break;
}
}
}
}
assert( rc!=SQLITE_OK || nCol>0 );
}
if( rc!=SQLITE_OK ) goto fts3_init_out;
@ -3045,6 +3062,8 @@ static int fts3ColumnMethod(
/* The extra column whose name is the same as the table.
** Return a blob which is a pointer to the cursor. */
sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
}else if( iCol==p->nColumn+2 && pCsr->pExpr ){
sqlite3_result_int64(pCtx, pCsr->iLangid);
}else{
/* The requested column is either a user column (one that contains
** indexed data), or the language-id column. */

14
manifest
View File

@ -1,5 +1,5 @@
C Add\stest\sfor\sFTS\s'rebuild'\scommand.
D 2012-03-02T16:18:21.845
C Fix\sproblems\swith\scombining\scontent=\sand\slanguageid=\sin\sa\ssingle\sfts4\stable.
D 2012-03-02T19:53:02.350
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a
F ext/fts3/fts3.c fd89caa4169520c32cf46ca5a62df6dd48201422
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb
F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239
@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
F test/fts4langid.test 343a65d54419b45a318a1acdb1c37749f26fdc79
F test/fts4langid.test be989b5cddcd7596b87232af193f6c4560a34272
F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
P 65fa693729a336e4d905ce72e6b9ccf4faa772bd
R 4e2c13b79ff592d73e478288d906b721
P 181bc35731f19c3e4497ba3338c209918d34ea69
R acfb20f690a18ac8d67e116ae8c76f7d
U dan
Z bcf6ed8432a11d3fdd53825eba08d4d7
Z b771fbbc3f5dd49e39970a5f917fc8b7

2
manifest.uuid
View File

@ -1 +1 @@
181bc35731f19c3e4497ba3338c209918d34ea69
22491e7bc38aee43819b888e04241cb6a6ef73a3

99
test/fts4langid.test
View File

@ -37,17 +37,17 @@ set ::testprefix fts4langid
#
# 2.3.* - Same as 2.1.*, after a 'rebuild' command.
#
# 3.* - Test that if one is provided, the tokenizer xLanguage method
# 3.* - Tests with content= tables. Both where there is a real
# underlying content table and where there is not.
#
#
# 4.* - Test that if one is provided, the tokenizer xLanguage method
# is called to configure the tokenizer before tokenizing query
# or document text.
#
# 4.* - Test the fts4aux table when the associated FTS4 table contains
# 5.* - Test the fts4aux table when the associated FTS4 table contains
# multiple languages.
#
# 5.* - Tests with content= tables. Both where there is a real
# underlying content table and where there is not.
#
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id);
@ -151,6 +151,11 @@ proc build_multilingual_db_1 {db} {
$db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) }
}
$db eval {
CREATE TABLE data(x, y, l);
INSERT INTO data(rowid, x, y, l) SELECT docid, x, y, l FROM t2;
}
}
proc rowid_list_set_langid {langid} {
@ -159,9 +164,9 @@ proc rowid_list_set_langid {langid} {
proc rowid_list {pattern} {
set langid $::rowid_list_langid
set res [list]
db eval {SELECT docid, x, y FROM t2 WHERE l = $langid ORDER BY docid ASC} {
db eval {SELECT rowid, x, y FROM data WHERE l = $langid ORDER BY rowid ASC} {
if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} {
lappend res $docid
lappend res $rowid
}
}
return $res
@ -235,7 +240,7 @@ do_test 2.0 {
build_multilingual_db_1 db
} {}
proc do_test_2 {tn query res_script} {
proc do_test_query1 {tn query res_script} {
for {set langid 0} {$langid < 10} {incr langid} {
rowid_list_set_langid $langid
set res [eval $res_script]
@ -243,17 +248,17 @@ proc do_test_2 {tn query res_script} {
set actual [
execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid}
]
do_test 2.$tn.$langid [list set {} $actual] $res
do_test $tn.$langid [list set {} $actual] $res
}
}
# Run some queries.
do_test_2 1.1 {delta} { rowid_list delta }
do_test_2 1.2 {"zero one two"} { rowid_list "zero one two" }
do_test_2 1.3 {zero one two} {
do_test_query1 2.1.1 {delta} { rowid_list delta }
do_test_query1 2.1.2 {"zero one two"} { rowid_list "zero one two" }
do_test_query1 2.1.3 {zero one two} {
and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
}
do_test_2 1.4 {"zero one" OR "one two"} {
do_test_query1 2.1.4 {"zero one" OR "one two"} {
or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
}
@ -264,12 +269,12 @@ do_execsql_test 2.2 {
INSERT INTO t2(t2) VALUES('optimize');
SELECT count(*) FROM t2_segdir;
} {9}
do_test_2 2.1 {delta} { rowid_list delta }
do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" }
do_test_2 2.3 {zero one two} {
do_test_query1 2.2.1 {delta} { rowid_list delta }
do_test_query1 2.2.2 {"zero one two"} { rowid_list "zero one two" }
do_test_query1 2.2.3 {zero one two} {
and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
}
do_test_2 2.4 {"zero one" OR "one two"} {
do_test_query1 2.2.4 {"zero one" OR "one two"} {
or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
}
@ -280,14 +285,64 @@ do_test 2.3 {
build_multilingual_db_1 db
execsql { INSERT INTO t2(t2) VALUES('rebuild') }
} {}
do_test_2 3.1 {delta} { rowid_list delta }
do_test_2 3.2 {"zero one two"} { rowid_list "zero one two" }
do_test_2 3.3 {zero one two} {
do_test_query1 2.3.1 {delta} { rowid_list delta }
do_test_query1 2.3.2 {"zero one two"} { rowid_list "zero one two" }
do_test_query1 2.3.3 {zero one two} {
and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
}
do_test_2 3.4 {"zero one" OR "one two"} {
do_test_query1 2.3.4 {"zero one" OR "one two"} {
or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
}
#-------------------------------------------------------------------------
# Test cases 3.*
#
do_test 3.0 {
reset_db
build_multilingual_db_1 db
execsql {
CREATE TABLE t3_data(l, x, y);
INSERT INTO t3_data(rowid, l, x, y) SELECT docid, l, x, y FROM t2;
DROP TABLE t2;
}
} {}
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE t2 USING fts4(content=t3_data, languageid=l);
INSERT INTO t2(t2) VALUES('rebuild');
}
do_test_query1 3.1.1 {delta} { rowid_list delta }
do_test_query1 3.1.2 {"zero one two"} { rowid_list "zero one two" }
do_test_query1 3.1.3 {zero one two} {
and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
}
do_test_query1 3.1.4 {"zero one" OR "one two"} {
or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
}
do_execsql_test 3.2.1 {
DROP TABLE t2;
CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l, content=nosuchtable);
}
do_execsql_test 3.2.2 {
INSERT INTO t2(docid, x, y, l) SELECT rowid, x, y, l FROM t3_data;
}
do_execsql_test 3.2.3 {
DROP TABLE t3_data;
}
do_test_query1 3.3.1 {delta} { rowid_list delta }
do_test_query1 3.3.2 {"zero one two"} { rowid_list "zero one two" }
do_test_query1 3.3.3 {zero one two} {
and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
}
do_test_query1 3.3.4 {"zero one" OR "one two"} {
or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
}
finish_test