Change the name of the "unicode" tokenizer to "unicode61" to emphasize that the case folding and separator-character identification routines are based on unicode version 6.1.
FossilOrigin-Name: 8f3e60aa2253f21bcee5d03982cfdd7f16c00060
This commit is contained in:
dan
parent
1c7016c9a5
commit
ab322bd21e
@ -11,7 +11,7 @@
|
||||
);
|
||||
|
||||
The built-in tokenizers (valid values to pass as <tokenizer name>) are
|
||||
"simple" and "porter".
|
||||
"simple", "porter" and "unicode".
|
||||
|
||||
<tokenizer-args> should consist of zero or more white-space separated
|
||||
arguments to pass to the selected tokenizer implementation. The
|
||||
|
||||
@ -3601,7 +3601,7 @@ int sqlite3Fts3Init(sqlite3 *db){
|
||||
if( rc==SQLITE_OK ){
|
||||
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|
||||
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
|
||||
|| sqlite3Fts3HashInsert(pHash, "unicode", 8, (void *)pUnicode)
|
||||
|| sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
|
||||
#ifdef SQLITE_ENABLE_ICU
|
||||
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
|
||||
#endif
|
||||
|
||||
16
manifest
16
manifest
@ -1,5 +1,5 @@
|
||||
C Add\sspecial\sfast\spaths\sto\ssqlite3FtsUnicodeTolower()\sand\sIsalnum()\sfor\scodepoints\sin\sthe\sASCII\srange.
|
||||
D 2012-05-25T19:50:12.903
|
||||
C Change\sthe\sname\sof\sthe\s"unicode"\stokenizer\sto\s"unicode61"\sto\semphasize\sthat\sthe\scase\sfolding\sand\sseparator-character\sidentification\sroutines\sare\sbased\son\sunicode\sversion\s6.1.
|
||||
D 2012-05-26T14:54:50.307
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -53,9 +53,9 @@ F ext/fts2/fts2_tokenizer1.c 0123d21078e053bd98fd6186c5c6dc6d67969f2e
|
||||
F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
|
||||
F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
|
||||
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
|
||||
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
|
||||
F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c 81c77264290b88ed80b7ad23ced152193eefae8f
|
||||
F ext/fts3/fts3.c aff512cbc4aea2c5a8d6b7614e81a7b5a4bb822c
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3Int.h 7b163fa22e7a625c404c424f2779a4d7b14c14ad
|
||||
F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
|
||||
@ -501,7 +501,7 @@ F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
|
||||
F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
|
||||
F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
|
||||
F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
|
||||
F test/fts4unicode.test 0627683f8ca06035d677d4becc2cd0dc57149ef7
|
||||
F test/fts4unicode.test 073546a151a40c591062d4e66934964a2c3731f8
|
||||
F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01
|
||||
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
|
||||
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
|
||||
@ -1004,7 +1004,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
|
||||
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
|
||||
F tool/warnings-clang.sh a8a0a3babda96dfb1ff51adda3cbbf3dfb7266c2
|
||||
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
|
||||
P 3dc567ef4702d9a63d78d11ff705cb7f7359f7a6
|
||||
R a4a319ece993f7d02e2811bac8d8bd2b
|
||||
P cf7b25d47687635a04f4347d45f135c686b9d758
|
||||
R 5906d3a9fcf2b2ea0b4312db10ed028e
|
||||
U dan
|
||||
Z e7946b1fde61e6ce4870009a94d8c133
|
||||
Z 9d1b06ce70a747334f00416ba78508e5
|
||||
|
||||
@ -1 +1 @@
|
||||
cf7b25d47687635a04f4347d45f135c686b9d758
|
||||
8f3e60aa2253f21bcee5d03982cfdd7f16c00060
|
||||
@ -20,7 +20,7 @@ set ::testprefix fts4unicode
|
||||
proc do_unicode_token_test {tn input res} {
|
||||
set input [string map {' ''} $input]
|
||||
uplevel [list do_execsql_test $tn "
|
||||
SELECT fts3_tokenizer_test('unicode', '$input');
|
||||
SELECT fts3_tokenizer_test('unicode61', '$input');
|
||||
" [list [list {*}$res]]]
|
||||
}
|
||||
|
||||
@ -40,5 +40,100 @@ do_unicode_token_test 1.7 "The\u00bfquick\u224ebrown\u2263fox" {
|
||||
0 the The 1 quick quick 2 brown brown 3 fox fox
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
set docs [list {
|
||||
Enhance the INSERT syntax to allow multiple rows to be inserted via the
|
||||
VALUES clause.
|
||||
} {
|
||||
Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
|
||||
} {
|
||||
Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
|
||||
} {
|
||||
Added the sqlite3_db_readonly() interface.
|
||||
} {
|
||||
Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
|
||||
ability to add new PRAGMA statements or to override built-in PRAGMAs.
|
||||
} {
|
||||
Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
|
||||
the same row that contains the maximum x value.
|
||||
} {
|
||||
Added support for the FTS4 languageid option.
|
||||
} {
|
||||
Documented support for the FTS4 content option. This feature has actually
|
||||
been in the code since version 3.7.9 but is only now considered to be
|
||||
officially supported.
|
||||
} {
|
||||
Pending statements no longer block ROLLBACK. Instead, the pending statement
|
||||
will return SQLITE_ABORT upon next access after the ROLLBACK.
|
||||
} {
|
||||
Improvements to the handling of CSV inputs in the command-line shell
|
||||
} {
|
||||
Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
|
||||
incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
|
||||
connected by OR.
|
||||
}]
|
||||
|
||||
set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS
|
||||
set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS
|
||||
set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS
|
||||
set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS
|
||||
set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS
|
||||
set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS
|
||||
set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS
|
||||
set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS
|
||||
set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS
|
||||
foreach k [array names map] {
|
||||
lappend mappings [string toupper $k] [lindex $map($k) 0]
|
||||
lappend mappings $k [lindex $map($k) 1]
|
||||
}
|
||||
proc mapdoc {doc} {
|
||||
set doc [regsub -all {[[:space:]]+} $doc " "]
|
||||
string map $::mappings [string trim $doc]
|
||||
}
|
||||
|
||||
do_test 2.0 {
|
||||
execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); }
|
||||
foreach doc $docs {
|
||||
set d [mapdoc $doc]
|
||||
execsql { INSERT INTO t2 VALUES($d) }
|
||||
}
|
||||
} {}
|
||||
|
||||
do_test 2.1 {
|
||||
set q [mapdoc "row"]
|
||||
execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
|
||||
} [list [mapdoc {
|
||||
Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
|
||||
the same row that contains the maximum x value.
|
||||
}]]
|
||||
|
||||
foreach {tn query snippet} {
|
||||
2 "row" {
|
||||
...returns the value of y on the same [row] that contains
|
||||
the maximum x value.
|
||||
}
|
||||
3 "ROW" {
|
||||
...returns the value of y on the same [row] that contains
|
||||
the maximum x value.
|
||||
}
|
||||
4 "rollback" {
|
||||
...[ROLLBACK]. Instead, the pending statement
|
||||
will return SQLITE_ABORT upon next access after the [ROLLBACK].
|
||||
}
|
||||
5 "rOllback" {
|
||||
...[ROLLBACK]. Instead, the pending statement
|
||||
will return SQLITE_ABORT upon next access after the [ROLLBACK].
|
||||
}
|
||||
6 "lang*" {
|
||||
Added support for the FTS4 [languageid] option.
|
||||
}
|
||||
} {
|
||||
do_test 2.$tn {
|
||||
set q [mapdoc $query]
|
||||
execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q }
|
||||
} [list [mapdoc $snippet]]
|
||||
}
|
||||
|
||||
finish_test
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user