mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Back out the change from [7fab1393c2b22b1f] that tries to convert invalid

Browse Source 
surrogate characters in UTF16 into the replacement character 0xfffd, as we
find that this breaks some software.

FossilOrigin-Name: 4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609
This commit is contained in:
drh 2020-05-20 15:02:04 +00:00
parent 5a8cd2e40c
commit 4f1315a432
4 changed files with 28 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
manifest
View File

@ -1,5 +1,5 @@
C Add\scompiler\shints\sto\sdisable\sTSAN\sfor\sthe\sroutines\sthat\saccess\sthe\s-shm\nfile\sheader\sin\sWAL\smode\susing\sa\sdouble-read\swith\smemory\sbarrier.
D 2020-05-19T15:51:10.216
C Back\sout\sthe\schange\sfrom\s[7fab1393c2b22b1f]\sthat\stries\sto\sconvert\sinvalid\nsurrogate\scharacters\sin\sUTF16\sinto\sthe\sreplacement\scharacter\s0xfffd,\sas\swe\nfind\sthat\sthis\sbreaks\ssome\ssoftware.
D 2020-05-20T15:02:04.035
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -602,7 +602,7 @@ F src/treeview.c 82c6391a3ba76215d4185fd4719a56ec4caf186a40c8a7b6e6ba4ae4467c274
F src/trigger.c 4ada1037cc99777f647a882cdacbd1a4deb6567b69daf02946286401b88cdc04
F src/update.c 3199098455830fc2d8c8fc4ae3ec2ea513eef64339ae9a7048db62b21169bc7a
F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78
F src/utf.c 95fb6e03a5ca679045c5adccd05380f0addccabef5911abddcb06af069500ab7
F src/utf.c d7a61c1dfdac3eb091d43341a674032dca5a34e122f78ef0b5bd2d5a31967dde
F src/util.c 3b6cedf7a0c69bd6e1acce832873952d416212d6293b18d03064e07d7a9b5118
F src/vacuum.c de9780b89fa4ee74c3534f60b94820e3179aca759ffc1338ee53cb4ea7693dd3
F src/vdbe.c e3dba0dee25bc92e871c13cac655260912b3be4abcc85b439259c9934b208da3
@ -1446,7 +1446,7 @@ F test/tkt-385a5b56b9.test 5204a7cba0e28c99df0acbf95af5e1af4d32965a7a14de6eccebf
F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678
F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7
F test/tkt-3a77c9714e.test 90e3e8455ee945a4076d4c44062b8845708af24a880355328fe7008f2047c9f0
F test/tkt-3fe897352e.test 6849fde0a87165ff83f54f5047af7c743d72af26908fadb90174f3294450b3f4
F test/tkt-3fe897352e.test 27e26eb0f1811aeba4d65aba43a4c52e99da5e70
F test/tkt-4a03edc4c8.test 91c0e135888cdc3d4eea82406a44b05c8c1648d0
F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a
F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407
@ -1866,7 +1866,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P cb0a18e64f8d81c2ada9f698faaf5ca68a0666687bf6f3abc860324cb1997463
R 8462379efa542a8195db58fd9ef94a35
P 3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde
R e941513d71fae9ce3b558d98a564d148
U drh
Z 5f8f2ff4f8e169f83e77c91d9875e042
Z db42a456dee4143dc7c71c6203f4a854

2
manifest.uuid
View File

@ -1 +1 @@
3117c1b5a9e348fd8d16ba9d03fdafaad8514567fb3403f72b86d6162ad40bde
4218c7b71fb6b227dbe4b852718584c150164af2d84e067cb810aa602554a609

16
src/utf.c
View File

@ -284,6 +284,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
c = *(zIn++);
c += (*(zIn++))<<8;
if( c>=0xd800 && c<0xe000 ){
#ifdef SQLITE_REPLACE_INVALID_UTF
if( c>=0xdc00 || zIn>=zTerm ){
c = 0xfffd;
}else{
@ -296,6 +297,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000;
}
}
#else
if( zIn<zTerm ){
int c2 = (*zIn++);
c2 += ((*zIn++)<<8);
c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);
}
#endif
}
WRITE_UTF8(z, c);
}
@ -305,6 +313,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
c = (*(zIn++))<<8;
c += *(zIn++);
if( c>=0xd800 && c<0xe000 ){
#ifdef SQLITE_REPLACE_INVALID_UTF
if( c>=0xdc00 || zIn>=zTerm ){
c = 0xfffd;
}else{
@ -317,6 +326,13 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000;
}
}
#else
if( zIn<zTerm ){
int c2 = ((*zIn++)<<8);
c2 += (*zIn++);
c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);
}
#endif
}
WRITE_UTF8(z, c);
}

8
test/tkt-3fe897352e.test
View File

@ -33,28 +33,28 @@ do_test tkt-3fe89-1.1 {
INSERT INTO t1 VALUES(hex_to_utf16be('D800'));
SELECT hex(x) FROM t1;
}
} {EFBFBD}
} {EDA080}
do_test tkt-3fe89-1.2 {
db eval {
DELETE FROM t1;
INSERT INTO t1 VALUES(hex_to_utf16le('00D8'));
SELECT hex(x) FROM t1;
}
} {EFBFBD}
} {EDA080}
do_test tkt-3fe89-1.3 {
db eval {
DELETE FROM t1;
INSERT INTO t1 VALUES(hex_to_utf16be('DFFF'));
SELECT hex(x) FROM t1;
}
} {EFBFBD}
} {EDBFBF}
do_test tkt-3fe89-1.4 {
db eval {
DELETE FROM t1;
INSERT INTO t1 VALUES(hex_to_utf16le('FFDF'));
SELECT hex(x) FROM t1;
}
} {EFBFBD}
} {EDBFBF}
finish_test