mirrors
/
sqlite
mirror of https://github.com/sqlite/sqlite
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FTS5 does not handle tokens that contain embedded nul characters. Prevent the trigram tokenizer from returning such tokens. Fix for [2ba5930b2]. 

FossilOrigin-Name: b1d048748c054575425a4bebf0c5d09962f9329d5ce6a978cf54e508b238584c
This commit is contained in:
dan 2020-10-03 14:36:06 +00:00
parent 7093a3be03
commit 95dca8d0cf
4 changed files with 22 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ext/fts5/fts5_tokenize.c
View File

@ -1333,11 +1333,13 @@ static int fts5TriTokenize(
const unsigned char *zNext;
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
zNext = zIn;
if( zIn<zEof ){
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
}else{
break;
}
@ -1345,6 +1347,7 @@ static int fts5TriTokenize(
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
WRITE_UTF8(zOut, iCode);
}else{

11
ext/fts5/test/fts5trigram.test
View File

@ -129,6 +129,17 @@ do_catchsql_test 3.3 {
CREATE VIRTUAL TABLE ttt USING fts5(c, "tokenize=trigram case_sensitive 1");
} {0 {}}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t0 USING fts5(b, tokenize = "trigram");
}
do_execsql_test 4.1 {
INSERT INTO t0 VALUES (x'000b01');
}
do_execsql_test 4.2 {
INSERT INTO t0(t0) VALUES('integrity-check');
}
finish_test

14
manifest
View File

@ -1,5 +1,5 @@
C Fix\ssome\stest\scode\sso\sthat\stestfixture\scan\sbuild\swith\sSQLITE_OMIT_WAL\sdefined.
D 2020-10-02T15:15:18.200
C FTS5\sdoes\snot\shandle\stokens\sthat\scontain\sembedded\snul\scharacters.\sPrevent\sthe\strigram\stokenizer\sfrom\sreturning\ssuch\stokens.\sFix\sfor\s[2ba5930b2].
D 2020-10-03T14:36:06.096
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -124,7 +124,7 @@ F ext/fts5/fts5_storage.c 58ba71e6cd3d43a5735815e7956ee167babb4d2cbfe20690517479
F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
F ext/fts5/fts5_test_tok.c f96c6e193c466711d6d7828d5f190407fe7ab897062d371426dd3036f01258e7
F ext/fts5/fts5_tokenize.c 5711f170065d23809afa97475c4adcd03387da043feb2fb2a1660fe366c01647
F ext/fts5/fts5_tokenize.c 6f47244681c670ec3c1364f19b2ec0cca191249ff3543755a65e1fc1df348061
F ext/fts5/fts5_unicode2.c 8bd0cd07396b74c1a05590e4070d635bccfc849812c305619f109e6c0485e250
F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
F ext/fts5/fts5_vocab.c 7a071833064dc8bca236c3c323e56aac36f583aa2c46ce916d52e31ce87462c9
@ -217,7 +217,7 @@ F ext/fts5/test/fts5synonym2.test b54cce5c34ec08ed616f646635538ae82e34a0e28f947e
F ext/fts5/test/fts5tok1.test ce6551e41ff56f30b69963577324624733bed0d1753589f06120d664d9cd45c9
F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43
F ext/fts5/test/fts5trigram.test 0a9ade9e808c388d13e9ea925a1bf42f3fa873a90dcd5969ef6e50bdd9483873
F ext/fts5/test/fts5trigram.test 29d13f2293899c8a9db216af55f6bf0df520459ea2952df1b7866302ef0d0dea
F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e
@ -1882,7 +1882,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P ad5ab24ebd557e7af1d92ab3fbcb3747c04da5ad4ed779fb6391dc94042687dd
R 1cdad7d96c36f2e03dd55b2f6f56718c
P dd009cd7aec3598e930806907601f4f0f9b0021ea99fa6c5e29e88f1246066ed
R 9c7acf6623596414564a0020768bc2dc
U dan
Z 9709ab64200eaa0e420a4dbfbec12e7d
Z 46788d8d61b39b3ddef723b54e3710a9

2
manifest.uuid
View File

@ -1 +1 @@
dd009cd7aec3598e930806907601f4f0f9b0021ea99fa6c5e29e88f1246066ed
b1d048748c054575425a4bebf0c5d09962f9329d5ce6a978cf54e508b238584c