mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Treat byte-order marks (BOMs) at the start of a token as whitespace.

Browse Source 
This enhancement is inspired by
[forum:/forumpost/ed8f696a20|forum post ed8f696a20].

FossilOrigin-Name: 3d55c21c167631f42d155aadec544e629bd078de9992aa5a74694d08bc52052b
This commit is contained in:
drh 2021-04-24 12:24:08 +00:00
parent 971ae9f66a
commit ba9ebc2d12
3 changed files with 24 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
manifest
View File

@ -1,5 +1,5 @@
C Fix\stestcase\saltertab-25.1\sdue\sto\scheck-in\s[c7909e8e0d0577c6]\sdisallowing\nALTER\sTABLE\son\seponymous\svirtual\stables. C Treat\sbyte-order\smarks\s(BOMs)\sat\sthe\sstart\sof\sa\stoken\sas\swhitespace.\nThis\senhancement\sis\sinspired\sby\n[forum:/forumpost/ed8f696a20|forum\spost\sed8f696a20].
D 2021-04-24T12:20:10.489 D 2021-04-24T12:24:08.900
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -607,7 +607,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f
F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
F src/tokenize.c 0b9c82fa628b5adce93e2bcaf935a24d43eb83344fb51551f7835526d0693fc4 F src/tokenize.c bae853ad129d1129c063de8630a3e99e306283bc40146f359b1bb91be2c08f1e
F src/treeview.c e483aeedf6f207000db1f90eb6abd816350493314c30e8749d319bdb9ab3b08c F src/treeview.c e483aeedf6f207000db1f90eb6abd816350493314c30e8749d319bdb9ab3b08c
F src/trigger.c f8493674f5c8f103c1a2cd0616af9dca85c7058450f9fe47cacd15cf5d512d52 F src/trigger.c f8493674f5c8f103c1a2cd0616af9dca85c7058450f9fe47cacd15cf5d512d52
F src/update.c b3abdaf4a314bbed238da69a6ca54c0f21262119389b412ee5778fffe62dd3cc F src/update.c b3abdaf4a314bbed238da69a6ca54c0f21262119389b412ee5778fffe62dd3cc
@ -1914,7 +1914,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P e7b4ffecc610c494ebd506977402ec48cc799780f96c6293c0ccf27697160aa1 P 32255e39cbde65492d88177464cee9e10cb20cf3105208416be131e2c89b63e1
R cf484e06e363477fe958c24db002cbc0 R 1d0e571ef291122bdf317bcd3d07015b
U drh U drh
Z bae3a8673bc5b7dab6af1040deb31bfe Z 9d760c4ee05493d61d6146a0ceb36fa0

2
manifest.uuid
View File

@ -1 +1 @@
32255e39cbde65492d88177464cee9e10cb20cf3105208416be131e2c89b63e1 3d55c21c167631f42d155aadec544e629bd078de9992aa5a74694d08bc52052b

25
src/tokenize.c
View File

@ -56,6 +56,7 @@
#define CC_ID 27 /* unicode characters usable in IDs */ #define CC_ID 27 /* unicode characters usable in IDs */
#define CC_ILLEGAL 28 /* Illegal character */ #define CC_ILLEGAL 28 /* Illegal character */
#define CC_NUL 29 /* 0x00 */ #define CC_NUL 29 /* 0x00 */
#define CC_BOM 30 /* First byte of UTF8 BOM: 0xEF 0xBB 0xBF */
static const unsigned char aiClass[] = { static const unsigned char aiClass[] = {
#ifdef SQLITE_ASCII #ifdef SQLITE_ASCII
@ -68,14 +69,14 @@ static const unsigned char aiClass[] = {
/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 9, 28, 28, 28, 2, /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 9, 28, 28, 28, 2,
/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 28, 10, 28, 25, 28, /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 28, 10, 28, 25, 28,
/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 8x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 9x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* Ax */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* Cx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* Dx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* Ex */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 30,
/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 /* Fx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27
#endif #endif
#ifdef SQLITE_EBCDIC #ifdef SQLITE_EBCDIC
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
@ -535,6 +536,14 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
i = 1; i = 1;
break; break;
} }
case CC_BOM: {
if( z[1]==0xbb && z[2]==0xbf ){
*tokenType = TK_SPACE;
return 3;
}
i = 1;
break;
}
case CC_NUL: { case CC_NUL: {
*tokenType = TK_ILLEGAL; *tokenType = TK_ILLEGAL;
return 0; return 0;