diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 6732edaece..dec7f8722c 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -800,13 +800,18 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); } - if( p->zLanguageid ) fts3Appendf(pRc, &zRet, ",langid"); + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } sqlite3_free(zFree); }else{ fts3Appendf(pRc, &zRet, "rowid"); for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } } fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", p->zDb, @@ -1215,8 +1220,20 @@ static int fts3InitVtab( sqlite3_free((void*)aCol); aCol = 0; rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; j0 ); } if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -3045,6 +3062,8 @@ static int fts3ColumnMethod( /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); }else{ /* The requested column is either a user column (one that contains ** indexed data), or the language-id column. */ diff --git a/manifest b/manifest index 268d6f64f6..15fcd20c9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfor\sFTS\s'rebuild'\scommand. -D 2012-03-02T16:18:21.845 +C Fix\sproblems\swith\scombining\scontent=\sand\slanguageid=\sin\sa\ssingle\sfts4\stable. +D 2012-03-02T19:53:02.350 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a +F ext/fts3/fts3.c fd89caa4169520c32cf46ca5a62df6dd48201422 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test 343a65d54419b45a318a1acdb1c37749f26fdc79 +F test/fts4langid.test be989b5cddcd7596b87232af193f6c4560a34272 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 65fa693729a336e4d905ce72e6b9ccf4faa772bd -R 4e2c13b79ff592d73e478288d906b721 +P 181bc35731f19c3e4497ba3338c209918d34ea69 +R acfb20f690a18ac8d67e116ae8c76f7d U dan -Z bcf6ed8432a11d3fdd53825eba08d4d7 +Z b771fbbc3f5dd49e39970a5f917fc8b7 diff --git a/manifest.uuid b/manifest.uuid index bdb54dd27f..def5beae28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -181bc35731f19c3e4497ba3338c209918d34ea69 \ No newline at end of file +22491e7bc38aee43819b888e04241cb6a6ef73a3 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index 059a56d08f..b9110cb768 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -37,17 +37,17 @@ set ::testprefix fts4langid # # 2.3.* - Same as 2.1.*, after a 'rebuild' command. # -# 3.* - Test that if one is provided, the tokenizer xLanguage method +# 3.* - Tests with content= tables. Both where there is a real +# underlying content table and where there is not. +# +# +# 4.* - Test that if one is provided, the tokenizer xLanguage method # is called to configure the tokenizer before tokenizing query # or document text. # -# 4.* - Test the fts4aux table when the associated FTS4 table contains +# 5.* - Test the fts4aux table when the associated FTS4 table contains # multiple languages. # -# 5.* - Tests with content= tables. Both where there is a real -# underlying content table and where there is not. -# - do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id); @@ -151,6 +151,11 @@ proc build_multilingual_db_1 {db} { $db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) } } + + $db eval { + CREATE TABLE data(x, y, l); + INSERT INTO data(rowid, x, y, l) SELECT docid, x, y, l FROM t2; + } } proc rowid_list_set_langid {langid} { @@ -159,9 +164,9 @@ proc rowid_list_set_langid {langid} { proc rowid_list {pattern} { set langid $::rowid_list_langid set res [list] - db eval {SELECT docid, x, y FROM t2 WHERE l = $langid ORDER BY docid ASC} { + db eval {SELECT rowid, x, y FROM data WHERE l = $langid ORDER BY rowid ASC} { if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} { - lappend res $docid + lappend res $rowid } } return $res @@ -235,7 +240,7 @@ do_test 2.0 { build_multilingual_db_1 db } {} -proc do_test_2 {tn query res_script} { +proc do_test_query1 {tn query res_script} { for {set langid 0} {$langid < 10} {incr langid} { rowid_list_set_langid $langid set res [eval $res_script] @@ -243,17 +248,17 @@ proc do_test_2 {tn query res_script} { set actual [ execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid} ] - do_test 2.$tn.$langid [list set {} $actual] $res + do_test $tn.$langid [list set {} $actual] $res } } # Run some queries. -do_test_2 1.1 {delta} { rowid_list delta } -do_test_2 1.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 1.3 {zero one two} { +do_test_query1 2.1.1 {delta} { rowid_list delta } +do_test_query1 2.1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.1.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 1.4 {"zero one" OR "one two"} { +do_test_query1 2.1.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } @@ -264,12 +269,12 @@ do_execsql_test 2.2 { INSERT INTO t2(t2) VALUES('optimize'); SELECT count(*) FROM t2_segdir; } {9} -do_test_2 2.1 {delta} { rowid_list delta } -do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 2.3 {zero one two} { +do_test_query1 2.2.1 {delta} { rowid_list delta } +do_test_query1 2.2.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.2.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 2.4 {"zero one" OR "one two"} { +do_test_query1 2.2.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } @@ -280,14 +285,64 @@ do_test 2.3 { build_multilingual_db_1 db execsql { INSERT INTO t2(t2) VALUES('rebuild') } } {} -do_test_2 3.1 {delta} { rowid_list delta } -do_test_2 3.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 3.3 {zero one two} { +do_test_query1 2.3.1 {delta} { rowid_list delta } +do_test_query1 2.3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.3.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 3.4 {"zero one" OR "one two"} { +do_test_query1 2.3.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } +#------------------------------------------------------------------------- +# Test cases 3.* +# +do_test 3.0 { + reset_db + build_multilingual_db_1 db + execsql { + CREATE TABLE t3_data(l, x, y); + INSERT INTO t3_data(rowid, l, x, y) SELECT docid, l, x, y FROM t2; + DROP TABLE t2; + } +} {} +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(content=t3_data, languageid=l); + INSERT INTO t2(t2) VALUES('rebuild'); +} + +do_test_query1 3.1.1 {delta} { rowid_list delta } +do_test_query1 3.1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 3.1.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 3.1.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +do_execsql_test 3.2.1 { + DROP TABLE t2; + CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l, content=nosuchtable); +} + +do_execsql_test 3.2.2 { + INSERT INTO t2(docid, x, y, l) SELECT rowid, x, y, l FROM t3_data; +} + +do_execsql_test 3.2.3 { + DROP TABLE t3_data; +} + +do_test_query1 3.3.1 {delta} { rowid_list delta } +do_test_query1 3.3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 3.3.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 3.3.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + + + finish_test