mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix problems in fts5 found by ASAN.

Browse Source 
FossilOrigin-Name: c564bf870106faef297594a51995619c80311d06bd5f8a0c7644f666f22ba576
This commit is contained in:
dan 2018-12-28 07:37:22 +00:00
parent 593c716781
commit b163b57212
11 changed files with 40 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ext/fts3/fts3_unicode2.c
View File

@ -200,7 +200,7 @@ static int remove_diacritic(int c, int bComplex){
'w', 'x', 'y', 'z', 'h', 't',
'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT,
'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT,
'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y',
'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@ -231,8 +231,8 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
(mask0 & ((unsigned int)1 << (c-768))) :
(mask1 & ((unsigned int)1 << (c-768-32)));
}

13
ext/fts3/unicode/mkunicode.tcl
View File

@ -63,14 +63,15 @@ proc print_rd {map} {
}
puts ""
puts " \};"
puts "#define HIBIT ((char)0x80)"
puts " char aChar\[\] = \{"
puts -nonewline " '\\0', "
set i 1
foreach c $aChar f $aFlag {
if { $f } {
set str "'$c'|0x80, "
set str "'$c'|HIBIT, "
} else {
set str "'$c'|0x00, "
set str "'$c', "
}
if {$c == ""} { set str "'\\0', " }
@ -134,8 +135,8 @@ proc print_isdiacritic {zFunc map} {
puts " if( c<$iFirst || c>$iLast ) return 0;"
puts " return (c < $iFirst+32) ?"
puts " (mask0 & (1 << (c-$iFirst))) :"
puts " (mask1 & (1 << (c-$iFirst-32)));"
puts " (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
puts " (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
puts "\}"
}
@ -699,7 +700,7 @@ proc print_categories {lMap} {
static u16 aFts5UnicodeMap[] = {$aMapArray};
static u16 aFts5UnicodeData[] = {$aDataArray};
int sqlite3Fts5UnicodeCategory(int iCode) {
int sqlite3Fts5UnicodeCategory(u32 iCode) {
int iRes = -1;
int iHi;
int iLo;
@ -782,7 +783,7 @@ proc print_test_categories {lMap} {
aArray[0] = 1;
}
c = sqlite3Fts5UnicodeCategory(i);
c = sqlite3Fts5UnicodeCategory((u32)i);
if( aArray[c]==0 ){
*piCode = i;
return 1;

2
ext/fts5/fts5Int.h
View File

@ -788,7 +788,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
int sqlite3Fts5UnicodeCatParse(const char*, u8*);
int sqlite3Fts5UnicodeCategory(int iCode);
int sqlite3Fts5UnicodeCategory(u32 iCode);
void sqlite3Fts5UnicodeAscii(u8*, u8*);
/*
** End of interface to code in fts5_unicode2.c.

2
ext/fts5/fts5_expr.c
View File

@ -2553,7 +2553,7 @@ static void fts5ExprIsAlnum(
sqlite3Fts5UnicodeCatParse("N*", aArr);
sqlite3Fts5UnicodeCatParse("Co", aArr);
iCode = sqlite3_value_int(apVal[0]);
sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]);
}
static void fts5ExprFold(

7
ext/fts5/fts5_index.c
View File

@ -3577,14 +3577,14 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
if( iId<=FTS5_MAX_SEGMENT ){
aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32);
}
}
}
for(i=0; aUsed[i]==0xFFFFFFFF; i++);
mask = aUsed[i];
for(iSegid=0; mask & (1 << iSegid); iSegid++);
for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++);
iSegid += 1 + i*32;
#ifdef SQLITE_DEBUG
@ -6282,8 +6282,7 @@ static void fts5DecodeFunction(
nSpace = n + FTS5_DATA_ZERO_PADDING;
a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
if( a==0 ) goto decode_out;
memcpy(a, aBlob, n);
if( n>0 ) memcpy(a, aBlob, n);
fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno);

2
ext/fts5/fts5_test_tok.c
View File

@ -378,7 +378,7 @@ static int fts5tokFilterMethod(
if( pCsr->zInput==0 ){
rc = SQLITE_NOMEM;
}else{
memcpy(pCsr->zInput, zByte, nByte);
if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte);
pCsr->zInput[nByte] = 0;
rc = pTab->tok.xTokenize(
pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb

4
ext/fts5/fts5_tokenize.c
View File

@ -268,7 +268,7 @@ static int fts5UnicodeAddExceptions(
if( iCode<128 ){
p->aTokenChar[iCode] = (unsigned char)bTokenChars;
}else{
bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
bToken = p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)];
assert( (bToken==0 || bToken==1) );
assert( (bTokenChars==0 || bTokenChars==1) );
if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
@ -429,7 +429,7 @@ static int fts5UnicodeCreate(
*/
static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
return (
p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
^ fts5UnicodeIsException(p, iCode)
);
}

10
ext/fts5/fts5_unicode2.c
View File

@ -69,7 +69,7 @@ static int fts5_remove_diacritic(int c, int bComplex){
'w', 'x', 'y', 'z', 'h', 't',
'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT,
'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT,
'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y',
'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@ -100,8 +100,8 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
(mask0 & ((unsigned int)1 << (c-768))) :
(mask1 & ((unsigned int)1 << (c-768-32)));
}
@ -249,6 +249,7 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
return ret;
}
int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
aArray[0] = 1;
switch( zCat[0] ){
@ -730,7 +731,7 @@ static u16 aFts5UnicodeData[] = {
34, 3074, 7692, 63, 63,
};
int sqlite3Fts5UnicodeCategory(int iCode) {
int sqlite3Fts5UnicodeCategory(u32 iCode) {
int iRes = -1;
int iHi;
int iLo;
@ -773,3 +774,4 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
iTbl++;
}
}

2
ext/fts5/test/fts5unicode3.test
View File

@ -21,7 +21,7 @@ ifcapable !fts5 {
}
proc fts3_unicode_path {file} {
file join [file dirname [info script]] .. .. fts3 unicode $file
file join .. [file dirname [info script]] .. .. fts3 unicode $file
}
source [fts3_unicode_path parseunicode.tcl]

28
manifest
View File

@ -1,5 +1,5 @@
C Fix\sanother\sproblem\swith\scorrupt\sdatabase\shandling\sin\sfts5.
D 2018-12-27T20:12:02.887
C Fix\sproblems\sin\sfts5\sfound\sby\sASAN.
D 2018-12-28T07:37:22.836
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6
@ -97,7 +97,7 @@ F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7
F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
F ext/fts3/fts3_unicode.c b1902e9ad47a6569fbb8ecb5ce52f20fe59b590d5c5e3bbdd56b10b03bdf632b
F ext/fts3/fts3_unicode2.c e49f9e015f239bf5faf2f4fa483bbf1b08a9978f0ad1f31159d952f8b8a10d08
F ext/fts3/fts3_unicode2.c faf0c750a1d0e9c9902f947212fc88f84e9450cddb3d1ffdbacbd0084b91d647
F ext/fts3/fts3_write.c e36d2f7e8f180e8030e92a5c2d09ccf87021afedcc5148a9d823b496667bf2f2
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
@ -105,24 +105,24 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87
F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl 2ea30d8122ccf1e33142c9cc8913d8cad9eb6668db359a228f10aeb37e2ab863
F ext/fts3/unicode/mkunicode.tcl 2315b3f8c83dcb09091009db01b321fb401d98d8c5489c147c1a7c5a54d21e20
F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h 4f5d19b7973dae23de368728f06d3eb1fe9f5cca2990366b40e9379996f35e61
F ext/fts5/fts5Int.h 39f12034b598df4e0f59bbe6cf03af03a905a534b04f182d155641a04e1eb797
F ext/fts5/fts5Int.h 8590e08a485c98a7e3075ed72abe3452fe944a9e58e63dfa51b732cced344cab
F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971ffd093
F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff
F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
F ext/fts5/fts5_expr.c 5aef080ba3c8947e22f38ce1ff9fe548e4a740e72b77241f35ed941ae128d2c7
F ext/fts5/fts5_expr.c 0eff5bad90838be83c5cc16837ca215a961abe05cfbf741a285ba62a79ad586f
F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55
F ext/fts5/fts5_index.c f3601579efeca9a859bf4cc82cbe99106fa8e3277a96b96c44ce35c7ceffb6d0
F ext/fts5/fts5_index.c c67a3bf1c185ff05937a9e3a01b8a6981f5a6add8da93911f461d59e57eea8d5
F ext/fts5/fts5_main.c 287a1a56580df304d7fa2fc1890f85b9cb6ac6b9e7c8af7dfa2151528db4b059
F ext/fts5/fts5_storage.c 5862f1b785a983acb8420281340f3f424896ab48f396f6fd8540787be7459139
F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
F ext/fts5/fts5_test_tok.c 80de1a4b1a3caa216c3be8862440f0117a8357dd9b7cfc5a2a2ce11fe6eb64ae
F ext/fts5/fts5_tokenize.c ca2b6a033794945ac505241a86b0aa978709c23aa2e6121984d3e3ede96003c8
F ext/fts5/fts5_unicode2.c 03bc8ebf44d825d0b7ac519f7b16139cf8e879ae7a27e62fb6326f0d68e8bdd3
F ext/fts5/fts5_test_tok.c 6a5574354ce61a98737e150fd4f7a002000db0cb4bcd655dc8694cb3e035381d
F ext/fts5/fts5_tokenize.c 240f849c9136103f77a67818deb172d8a7564cbefaf00ec7b2fd04ab17d6d66c
F ext/fts5/fts5_unicode2.c 3f1bad6f2924ad13e25408f00301edd12d4216388a11daa518cb28f79b337f5e
F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
F ext/fts5/fts5_vocab.c fbe38044889b2d2d99babeeef239c620fb0332bb928a84506ac748d81500b354
F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
@ -211,7 +211,7 @@ F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d21537
F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e
F ext/fts5/test/fts5unicode3.test 9cbc82e2b02e2e3b7504103580c90f095e07fe8230b1951a9ed7558717b5feb7
F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c75ae10f4afc6faa0744
F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687
F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d
F ext/fts5/test/fts5update.test 0737876e20e97a6a6abf45de19fc99315727bcee6a83fadcada1cc080b9aa8f0
@ -1794,7 +1794,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P d9e4f9574659c6285deba3dca68eca9989e615971cea3a901ffcda12fb4ee36f
R 308e6395f3ea79d50b990f4e574e128e
P fb0d7fba07865ff557bfcdacfda7b625d099ac8b4dcbe0c08cd2d4b84a068bb9
R 31f0c322d87dce0c17eac6b9b39fdaff
U dan
Z d6ac7bbdbfe101fd5a8ad83d11fdc195
Z f209c1d4c450aa64c337c8ad21da7144

2
manifest.uuid
View File

@ -1 +1 @@
fb0d7fba07865ff557bfcdacfda7b625d099ac8b4dcbe0c08cd2d4b84a068bb9
c564bf870106faef297594a51995619c80311d06bd5f8a0c7644f666f22ba576