If SQLITE_DISABLE_FTS3_UNICODE is defined, do not build the "unicode61" tokenizer.
FossilOrigin-Name: e71495a817b479bc23c5403d99255e3f098eb054
This commit is contained in:
dan
parent
501c74d3e1
commit
7946c53009
@ -3554,7 +3554,9 @@ static void hashDestroy(void *p){
|
||||
*/
|
||||
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
||||
void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
|
||||
#endif
|
||||
#ifdef SQLITE_ENABLE_ICU
|
||||
void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
||||
#endif
|
||||
@ -3570,13 +3572,19 @@ int sqlite3Fts3Init(sqlite3 *db){
|
||||
Fts3Hash *pHash = 0;
|
||||
const sqlite3_tokenizer_module *pSimple = 0;
|
||||
const sqlite3_tokenizer_module *pPorter = 0;
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
const sqlite3_tokenizer_module *pUnicode = 0;
|
||||
#endif
|
||||
|
||||
#ifdef SQLITE_ENABLE_ICU
|
||||
const sqlite3_tokenizer_module *pIcu = 0;
|
||||
sqlite3Fts3IcuTokenizerModule(&pIcu);
|
||||
#endif
|
||||
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
sqlite3Fts3UnicodeTokenizer(&pUnicode);
|
||||
#endif
|
||||
|
||||
#ifdef SQLITE_TEST
|
||||
rc = sqlite3Fts3InitTerm(db);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
@ -3587,7 +3595,6 @@ int sqlite3Fts3Init(sqlite3 *db){
|
||||
|
||||
sqlite3Fts3SimpleTokenizerModule(&pSimple);
|
||||
sqlite3Fts3PorterTokenizerModule(&pPorter);
|
||||
sqlite3Fts3UnicodeTokenizer(&pUnicode);
|
||||
|
||||
/* Allocate and initialise the hash-table used to store tokenizers. */
|
||||
pHash = sqlite3_malloc(sizeof(Fts3Hash));
|
||||
@ -3601,7 +3608,10 @@ int sqlite3Fts3Init(sqlite3 *db){
|
||||
if( rc==SQLITE_OK ){
|
||||
if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
|
||||
|| sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
|
||||
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
|| sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
|
||||
#endif
|
||||
#ifdef SQLITE_ENABLE_ICU
|
||||
|| (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
|
||||
#endif
|
||||
|
||||
@ -13,6 +13,8 @@
|
||||
** Implementation of the "unicode" full-text-search tokenizer.
|
||||
*/
|
||||
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
|
||||
#include "fts3Int.h"
|
||||
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
|
||||
|
||||
@ -241,3 +243,4 @@ void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){
|
||||
}
|
||||
|
||||
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
|
||||
#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
** DO NOT EDIT THIS MACHINE GENERATED FILE.
|
||||
*/
|
||||
|
||||
#ifndef SQLITE_DISABLE_FTS3_UNICODE
|
||||
|
||||
#include <assert.h>
|
||||
|
||||
/*
|
||||
@ -289,3 +291,4 @@ int sqlite3FtsUnicodeTolower(int c){
|
||||
|
||||
return ret;
|
||||
}
|
||||
#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
|
||||
|
||||
@ -507,6 +507,8 @@ proc print_fileheader {} {
|
||||
*/
|
||||
}]
|
||||
puts ""
|
||||
puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
|
||||
puts ""
|
||||
puts "#include <assert.h>"
|
||||
puts ""
|
||||
}
|
||||
@ -567,3 +569,5 @@ if {$::generate_test_code} {
|
||||
print_test_main
|
||||
}
|
||||
|
||||
puts "#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */"
|
||||
|
||||
|
||||
24
manifest
24
manifest
@ -1,5 +1,5 @@
|
||||
C Change\sthe\sformat\sof\sthe\stables\sused\sby\ssqlite3FtsUnicodeTolower()\sto\smake\sthem\sa\slittle\ssmaller.
|
||||
D 2012-05-26T17:57:02.187
|
||||
C If\sSQLITE_DISABLE_FTS3_UNICODE\sis\sdefined,\sdo\snot\sbuild\sthe\s"unicode61"\stokenizer.
|
||||
D 2012-05-26T18:28:14.350
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -55,7 +55,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
|
||||
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
|
||||
F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c aff512cbc4aea2c5a8d6b7614e81a7b5a4bb822c
|
||||
F ext/fts3/fts3.c 41824d0db7d244ca335ce98162df1244863a05c4
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3Int.h 7b163fa22e7a625c404c424f2779a4d7b14c14ad
|
||||
F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
|
||||
@ -70,15 +70,15 @@ F ext/fts3/fts3_test.c 348f7d08cae05285794e23dc4fe8b8fdf66e264a
|
||||
F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
|
||||
F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
|
||||
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
|
||||
F ext/fts3/fts3_unicode.c f487c6ef0ac2cc35aec3c3636c0a35483da6453c
|
||||
F ext/fts3/fts3_unicode2.c e43024fe05db9316b7825bfce809479fad6ce44e
|
||||
F ext/fts3/fts3_unicode.c 76b6f6fe6e86acd75b08272502fae74a13cef310
|
||||
F ext/fts3/fts3_unicode2.c 46ff2289f52e661ef030b4aed4c9e5b6132b6ea9
|
||||
F ext/fts3/fts3_write.c 6a6391d6b01114f885e24e1f66bbc11ffba0e9e2
|
||||
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
|
||||
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
|
||||
F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
|
||||
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
|
||||
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
|
||||
F ext/fts3/unicode/mkunicode.tcl 2775280062b6595dd18e44ddef6327239ab290ab
|
||||
F ext/fts3/unicode/mkunicode.tcl de64862abd50307c6e40622970eb9c8f2f14ed0e
|
||||
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
|
||||
F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a
|
||||
F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
|
||||
@ -200,7 +200,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad
|
||||
F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e
|
||||
F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de
|
||||
F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2
|
||||
F src/test_config.c 0de329e736eb4aa5845069bed630e5c72f012264
|
||||
F src/test_config.c d2da9f1490c38d9b9cb52f399b6adb81d538273a
|
||||
F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094
|
||||
F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
|
||||
F src/test_func.c 090f2c3339e85c2c964435f99aed6f3da9d59525
|
||||
@ -482,7 +482,7 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
|
||||
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
|
||||
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
|
||||
F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660
|
||||
F test/fts3fault2.test 9e3a0a0a9b0e29623e3fa9f96df346324d18fd48
|
||||
F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887
|
||||
F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
|
||||
F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be
|
||||
F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6
|
||||
@ -501,7 +501,7 @@ F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
|
||||
F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
|
||||
F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
|
||||
F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
|
||||
F test/fts4unicode.test dd0b67a2edde62eb79c34345968f378aaf4dec10
|
||||
F test/fts4unicode.test c812e9cf843e26ba633f58b36a2629f878af20fd
|
||||
F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01
|
||||
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
|
||||
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
|
||||
@ -1004,7 +1004,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
|
||||
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
|
||||
F tool/warnings-clang.sh a8a0a3babda96dfb1ff51adda3cbbf3dfb7266c2
|
||||
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
|
||||
P 07d3ea8a3cb179fab6c48934fc6751f53b507d36
|
||||
R fbb3511eb94203c7d1dd12033a92a0a8
|
||||
P b89d3834f6690073fca0fc22c18afa1fb280ea7d
|
||||
R 721b18aaca6e897933f70f12a0639abb
|
||||
U dan
|
||||
Z d24d4148848cb3328b6f3577a1e7fca3
|
||||
Z f1ca3c8a69800dd8f2afaacbafb654ad
|
||||
|
||||
@ -1 +1 @@
|
||||
b89d3834f6690073fca0fc22c18afa1fb280ea7d
|
||||
e71495a817b479bc23c5403d99255e3f098eb054
|
||||
@ -307,6 +307,12 @@ static void set_options(Tcl_Interp *interp){
|
||||
Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
|
||||
#endif
|
||||
|
||||
#if !defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_DISABLE_FTS3_UNICODE)
|
||||
Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY);
|
||||
#else
|
||||
Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY);
|
||||
#endif
|
||||
|
||||
#ifdef SQLITE_OMIT_GET_TABLE
|
||||
Tcl_SetVar2(interp, "sqlite_options", "gettable", "0", TCL_GLOBAL_ONLY);
|
||||
#else
|
||||
|
||||
@ -131,26 +131,28 @@ do_faultsim_test 4.1 -prep {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
do_test 5.0 {
|
||||
faultsim_delete_and_reopen
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
|
||||
ifcapable fts3_unicode {
|
||||
do_test 5.0 {
|
||||
faultsim_delete_and_reopen
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
|
||||
}
|
||||
faultsim_save_and_close
|
||||
} {}
|
||||
|
||||
do_faultsim_test 5.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
db eval {SELECT * FROM sqlite_master}
|
||||
} -body {
|
||||
execsql { INSERT INTO ft VALUES('the quick brown fox'); }
|
||||
execsql { INSERT INTO ft VALUES(
|
||||
'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
|
||||
);
|
||||
}
|
||||
execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
|
||||
} -test {
|
||||
faultsim_test_result {0 {1 2}}
|
||||
}
|
||||
faultsim_save_and_close
|
||||
} {}
|
||||
|
||||
do_faultsim_test 5.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
db eval {SELECT * FROM sqlite_master}
|
||||
} -body {
|
||||
execsql { INSERT INTO ft VALUES('the quick brown fox'); }
|
||||
execsql { INSERT INTO ft VALUES(
|
||||
'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
|
||||
);
|
||||
}
|
||||
execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
|
||||
} -test {
|
||||
faultsim_test_result {0 {1 2}}
|
||||
}
|
||||
|
||||
finish_test
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
|
||||
set testdir [file dirname $argv0]
|
||||
source $testdir/tester.tcl
|
||||
ifcapable !fts3 { finish_test ; return }
|
||||
ifcapable !fts3_unicode { finish_test ; return }
|
||||
set ::testprefix fts4unicode
|
||||
|
||||
proc do_unicode_token_test {tn input res} {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user