A new approach for UTF-8 translation. (CVS 4004)
FossilOrigin-Name: 6c8ad2790eaede90b3f1ef62614e667178b2a8c4
This commit is contained in:
drh
parent
eab7f3fe8d
commit
4a919118d3
16
manifest
16
manifest
@ -1,5 +1,5 @@
|
||||
C Do\snot\sallow\sbytes\s0x80\sthrough\s0xBF\sto\sbe\sthe\sfirst\scharacter\sof\san\nidentifer\sbecause\sno\svalid\sUTF-8\scharacter\scan\sbegin\swith\sthose\sbytes.\nIf\swe\sallowed\san\sidentifier\sto\sbegin\swith\sone\sof\sthose\sbytes,\sthen\sthe\nsubstr()\sfunction\sin\sALTER\sTABLE\swill\snot\swork\scorrectly.\s(CVS\s4003)
|
||||
D 2007-05-15T09:00:15
|
||||
C A\snew\sapproach\sfor\sUTF-8\stranslation.\s(CVS\s4004)
|
||||
D 2007-05-15T11:55:09
|
||||
F Makefile.in 87b200ad9970907f76df734d29dff3d294c10935
|
||||
F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
|
||||
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
|
||||
@ -71,7 +71,7 @@ F src/date.c 6049db7d5a8fdf2c677ff7d58fa31d4f6593c988
|
||||
F src/delete.c 5c0d89b3ef7d48fe1f5124bfe8341f982747fe29
|
||||
F src/experimental.c 1b2d1a6cd62ecc39610e97670332ca073c50792b
|
||||
F src/expr.c 436f1d3e5addf95c195016b518cd2f44a6f5f081
|
||||
F src/func.c cdbe36f2a71b7e6835189414c2fb626a339e300f
|
||||
F src/func.c fc1524fd6097b19c54cc4555e3ea724eec628e2c
|
||||
F src/hash.c 67b23e14f0257b69a3e8aa663e4eeadc1a2b6fd5
|
||||
F src/hash.h 1b3f7e2609141fd571f62199fc38687d262e9564
|
||||
F src/insert.c e595ca26805dfb3a9ebaabc28e7947c479f3b14d
|
||||
@ -104,7 +104,7 @@ F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96
|
||||
F src/shell.c d07ae326b3815d80f71c69b3c7584382e47f6447
|
||||
F src/sqlite.h.in aa3c0c02883663944db09ee79f3165d3c4ad7c47
|
||||
F src/sqlite3ext.h 7d0d363ea7327e817ef0dfe1b7eee1f171b72890
|
||||
F src/sqliteInt.h c31c9526bc602c3c71ddc45c548e987530826f11
|
||||
F src/sqliteInt.h 591f857fbf89a2a30f1d7c30018abb4596c4fc71
|
||||
F src/table.c a8de75bcedf84d4060d804264b067ab3b1a3561d
|
||||
F src/tclsqlite.c f425c7583665ef78dd8397b2de0b8e0028e80ce2
|
||||
F src/test1.c 84c841e1088f743200b87581506e93f70344bd32
|
||||
@ -129,7 +129,7 @@ F src/test_tclvar.c 315e77c17f128ff8c06b38c08617fd07c825a95b
|
||||
F src/tokenize.c 9aa8e3f06f56a700ef498582dae431be3d5c4f4c
|
||||
F src/trigger.c 420192efe3e6f03addf7897c60c3c8bf913d3493
|
||||
F src/update.c 3359041db390a8f856d67272f299600e2104f350
|
||||
F src/utf.c be7c64eed83fa3c01e0c42905e1c311dcd1be704
|
||||
F src/utf.c 3fe21361e83cfcf5c18764d8d5c59fffdf9144db
|
||||
F src/util.c 4f6bbcec2b2b1884d652b82c9f8949ede4618d68
|
||||
F src/vacuum.c 8bd895d29e7074e78d4e80f948e35ddc9cf2beef
|
||||
F src/vdbe.c 5deb4cdccd57065ccf8a2e5c704e8473c90d204b
|
||||
@ -491,7 +491,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
|
||||
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
|
||||
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
|
||||
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
|
||||
P 51eeae7b7ad5a0fbdd9d4418120dbdb6cd577cd5
|
||||
R 710324b40f5e54ea717a1562c7c07a4d
|
||||
P 252810424d8c4dcd19b369d62027094df7cf0bcc
|
||||
R 8b16cbfea1aee4c8ac49aefcccb7ea15
|
||||
U drh
|
||||
Z 6fd77513c09320f6a39989f1ee4db43a
|
||||
Z 1d3ef6d9d156a8b50e6d1607d4475644
|
||||
|
||||
@ -1 +1 @@
|
||||
252810424d8c4dcd19b369d62027094df7cf0bcc
|
||||
6c8ad2790eaede90b3f1ef62614e667178b2a8c4
|
||||
66
src/func.c
66
src/func.c
@ -16,7 +16,7 @@
|
||||
** sqliteRegisterBuildinFunctions() found at the bottom of the file.
|
||||
** All other code has file scope.
|
||||
**
|
||||
** $Id: func.c,v 1.156 2007/05/15 01:13:47 drh Exp $
|
||||
** $Id: func.c,v 1.157 2007/05/15 11:55:09 drh Exp $
|
||||
*/
|
||||
#include "sqliteInt.h"
|
||||
#include <ctype.h>
|
||||
@ -103,7 +103,11 @@ static void lengthFunc(
|
||||
case SQLITE_TEXT: {
|
||||
const unsigned char *z = sqlite3_value_text(argv[0]);
|
||||
if( z==0 ) return;
|
||||
for(len=0; *z; z++){ if( (0xc0&*z)!=0x80 ) len++; }
|
||||
len = 0;
|
||||
while( *z ){
|
||||
len++;
|
||||
SQLITE_SKIP_UTF8(z);
|
||||
}
|
||||
sqlite3_result_int(context, len);
|
||||
break;
|
||||
}
|
||||
@ -162,7 +166,6 @@ static void substrFunc(
|
||||
){
|
||||
const unsigned char *z;
|
||||
const unsigned char *z2;
|
||||
int i;
|
||||
int len;
|
||||
int p0type;
|
||||
i64 p1, p2;
|
||||
@ -176,7 +179,10 @@ static void substrFunc(
|
||||
}else{
|
||||
z = sqlite3_value_text(argv[0]);
|
||||
if( z==0 ) return;
|
||||
for(len=0, z2=z; *z2; z2++){ if( (0xc0&*z2)!=0x80 ) len++; }
|
||||
len = 0;
|
||||
for(z2=z; *z2; len++){
|
||||
SQLITE_SKIP_UTF8(z2);
|
||||
}
|
||||
}
|
||||
p1 = sqlite3_value_int(argv[1]);
|
||||
p2 = sqlite3_value_int(argv[2]);
|
||||
@ -193,16 +199,14 @@ static void substrFunc(
|
||||
p2 = len-p1;
|
||||
}
|
||||
if( p0type!=SQLITE_BLOB ){
|
||||
for(i=0; i<p1 && z[i]; i++){
|
||||
if( (z[i]&0xc0)==0x80 ) p1++;
|
||||
while( *z && p1 ){
|
||||
SQLITE_SKIP_UTF8(z);
|
||||
p1--;
|
||||
}
|
||||
while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p1++; }
|
||||
for(; i<p1+p2 && z[i]; i++){
|
||||
if( (z[i]&0xc0)==0x80 ) p2++;
|
||||
for(z2=z; *z2 && p2; p2--){
|
||||
SQLITE_SKIP_UTF8(z2);
|
||||
}
|
||||
while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p2++; }
|
||||
if( p2<0 ) p2 = 0;
|
||||
sqlite3_result_text(context, (char*)&z[p1], p2, SQLITE_TRANSIENT);
|
||||
sqlite3_result_text(context, (char*)z, z2-z, SQLITE_TRANSIENT);
|
||||
}else{
|
||||
if( p2<0 ) p2 = 0;
|
||||
sqlite3_result_blob(context, (char*)&z[p1], p2, SQLITE_TRANSIENT);
|
||||
@ -389,13 +393,13 @@ static const struct compareInfo likeInfoNorm = { '%', '_', 0, 1 };
|
||||
static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 };
|
||||
|
||||
/*
|
||||
** X is a pointer to the first byte of a UTF-8 character. Increment
|
||||
** X so that it points to the next character. This only works right
|
||||
** if X points to a well-formed UTF-8 string.
|
||||
** Read a single UTF-8 character and return its value.
|
||||
*/
|
||||
#define sqliteNextChar(X) while( (0xc0&*++(X))==0x80 ){}
|
||||
#define sqliteCharVal(X) sqlite3ReadUtf8(X)
|
||||
|
||||
u32 sqlite3ReadUtf8(const unsigned char *z){
|
||||
u32 c;
|
||||
SQLITE_READ_UTF8(z, c);
|
||||
return c;
|
||||
}
|
||||
|
||||
/*
|
||||
** Compare two UTF-8 strings for equality where the first string can
|
||||
@ -446,20 +450,20 @@ static int patternCompare(
|
||||
while( (c=zPattern[1]) == matchAll || c == matchOne ){
|
||||
if( c==matchOne ){
|
||||
if( *zString==0 ) return 0;
|
||||
sqliteNextChar(zString);
|
||||
SQLITE_SKIP_UTF8(zString);
|
||||
}
|
||||
zPattern++;
|
||||
}
|
||||
if( c && esc && sqlite3ReadUtf8(&zPattern[1])==esc ){
|
||||
u8 const *zTemp = &zPattern[1];
|
||||
sqliteNextChar(zTemp);
|
||||
SQLITE_SKIP_UTF8(zTemp);
|
||||
c = *zTemp;
|
||||
}
|
||||
if( c==0 ) return 1;
|
||||
if( c==matchSet ){
|
||||
assert( esc==0 ); /* This is GLOB, not LIKE */
|
||||
while( *zString && patternCompare(&zPattern[1],zString,pInfo,esc)==0 ){
|
||||
sqliteNextChar(zString);
|
||||
SQLITE_SKIP_UTF8(zString);
|
||||
}
|
||||
return *zString!=0;
|
||||
}else{
|
||||
@ -473,20 +477,20 @@ static int patternCompare(
|
||||
}
|
||||
if( c2==0 ) return 0;
|
||||
if( patternCompare(&zPattern[1],zString,pInfo,esc) ) return 1;
|
||||
sqliteNextChar(zString);
|
||||
SQLITE_SKIP_UTF8(zString);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}else if( !prevEscape && c==matchOne ){
|
||||
if( *zString==0 ) return 0;
|
||||
sqliteNextChar(zString);
|
||||
SQLITE_SKIP_UTF8(zString);
|
||||
zPattern++;
|
||||
}else if( c==matchSet ){
|
||||
int prior_c = 0;
|
||||
assert( esc==0 ); /* This only occurs for GLOB, not LIKE */
|
||||
seen = 0;
|
||||
invert = 0;
|
||||
c = sqliteCharVal(zString);
|
||||
c = sqlite3ReadUtf8(zString);
|
||||
if( c==0 ) return 0;
|
||||
c2 = *++zPattern;
|
||||
if( c2=='^' ){ invert = 1; c2 = *++zPattern; }
|
||||
@ -494,10 +498,10 @@ static int patternCompare(
|
||||
if( c==']' ) seen = 1;
|
||||
c2 = *++zPattern;
|
||||
}
|
||||
while( (c2 = sqliteCharVal(zPattern))!=0 && c2!=']' ){
|
||||
while( (c2 = sqlite3ReadUtf8(zPattern))!=0 && c2!=']' ){
|
||||
if( c2=='-' && zPattern[1]!=']' && zPattern[1]!=0 && prior_c>0 ){
|
||||
zPattern++;
|
||||
c2 = sqliteCharVal(zPattern);
|
||||
c2 = sqlite3ReadUtf8(zPattern);
|
||||
if( c>=prior_c && c<=c2 ) seen = 1;
|
||||
prior_c = 0;
|
||||
}else if( c==c2 ){
|
||||
@ -506,14 +510,14 @@ static int patternCompare(
|
||||
}else{
|
||||
prior_c = c2;
|
||||
}
|
||||
sqliteNextChar(zPattern);
|
||||
SQLITE_SKIP_UTF8(zPattern);
|
||||
}
|
||||
if( c2==0 || (seen ^ invert)==0 ) return 0;
|
||||
sqliteNextChar(zString);
|
||||
SQLITE_SKIP_UTF8(zString);
|
||||
zPattern++;
|
||||
}else if( esc && !prevEscape && sqlite3ReadUtf8(zPattern)==esc){
|
||||
prevEscape = 1;
|
||||
sqliteNextChar(zPattern);
|
||||
SQLITE_SKIP_UTF8(zPattern);
|
||||
}else{
|
||||
if( noCase ){
|
||||
if( sqlite3UpperToLower[c] != sqlite3UpperToLower[*zString] ) return 0;
|
||||
@ -851,7 +855,7 @@ static void trimFunc(
|
||||
}else{
|
||||
const unsigned char *z;
|
||||
for(z=zCharSet, nChar=0; *z; nChar++){
|
||||
sqliteNextChar(z);
|
||||
SQLITE_SKIP_UTF8(z);
|
||||
}
|
||||
if( nChar>0 ){
|
||||
azChar = sqlite3_malloc( nChar*(sizeof(char*)+1) );
|
||||
@ -861,7 +865,7 @@ static void trimFunc(
|
||||
aLen = (unsigned char*)&azChar[nChar];
|
||||
for(z=zCharSet, nChar=0; *z; nChar++){
|
||||
azChar[nChar] = z;
|
||||
sqliteNextChar(z);
|
||||
SQLITE_SKIP_UTF8(z);
|
||||
aLen[nChar] = z - azChar[nChar];
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
*************************************************************************
|
||||
** Internal interface definitions for SQLite.
|
||||
**
|
||||
** @(#) $Id: sqliteInt.h,v 1.566 2007/05/12 12:08:51 drh Exp $
|
||||
** @(#) $Id: sqliteInt.h,v 1.567 2007/05/15 11:55:09 drh Exp $
|
||||
*/
|
||||
#ifndef _SQLITEINT_H_
|
||||
#define _SQLITEINT_H_
|
||||
@ -1536,6 +1536,63 @@ typedef struct {
|
||||
*/
|
||||
extern int sqlite3_always_code_trigger_setup;
|
||||
|
||||
/*
|
||||
** A lookup table used by the SQLITE_READ_UTF8 macro. The definition
|
||||
** is in utf.c.
|
||||
*/
|
||||
extern const unsigned char sqlite3UtfTrans1[];
|
||||
|
||||
/*
|
||||
** Macros for reading UTF8 characters.
|
||||
**
|
||||
** SQLITE_READ_UTF8(x,c) reads a single UTF8 value out of x and writes
|
||||
** that value into c. The type of x must be unsigned char*. The type
|
||||
** of c must be unsigned int.
|
||||
**
|
||||
** SQLITE_SKIP_UTF8(x) advances x forward by one character. The type of
|
||||
** x must be unsigned char*.
|
||||
**
|
||||
** Notes On Invalid UTF-8:
|
||||
**
|
||||
** * These macros never allow a 7-bit character (0x00 through 0x7f) to
|
||||
** be encoded as a multi-byte character. Any multi-byte character that
|
||||
** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd.
|
||||
**
|
||||
** * These macros never allow a UTF16 surragate value to be encoded.
|
||||
** If a multi-byte character attempts to encode a value between
|
||||
** 0xd800 and 0xe000 then it is rendered as 0xfffd.
|
||||
**
|
||||
** * Bytes in the range of 0x80 through 0xbf which occur as the first
|
||||
** byte of a character are interpreted as single-byte characters
|
||||
** and rendered as themselves even though they are technically
|
||||
** invalid characters.
|
||||
**
|
||||
** * These routines accept an infinite number of different UTF8 encodings
|
||||
** for unicode values 0x80 and greater. They do not change over-length
|
||||
** encodings to 0xfffd as some systems recommend.
|
||||
**
|
||||
*/
|
||||
#define SQLITE_READ_UTF8(zIn, c) { \
|
||||
c = *(zIn++); \
|
||||
if( c>=0xc0 ){ \
|
||||
c = sqlite3UtfTrans1[c-0xc0]; \
|
||||
while( (*zIn & 0xc0)==0x80 ){ \
|
||||
c = (c<<6) + (0x3f & *(zIn++)); \
|
||||
} \
|
||||
if( c<0x80 \
|
||||
|| (c&0xFFFFF800)==0xD800 \
|
||||
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
|
||||
} \
|
||||
}
|
||||
#define SQLITE_SKIP_UTF8(zIn) { \
|
||||
if( (*(zIn++))>=0xc0 ){ \
|
||||
while( (*zIn & 0xc0)==0x80 ){ zIn++; } \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
** The SQLITE_CORRUPT_BKPT macro can be either a constant (for production
|
||||
** builds) or a function call (for debugging). If it is a function call,
|
||||
@ -1753,7 +1810,7 @@ int sqlite3GetInt32(const char *, int*);
|
||||
int sqlite3FitsIn64Bits(const char *);
|
||||
int sqlite3Utf16ByteLen(const void *pData, int nChar);
|
||||
int sqlite3Utf8CharLen(const char *pData, int nByte);
|
||||
int sqlite3ReadUtf8(const unsigned char *);
|
||||
u32 sqlite3ReadUtf8(const unsigned char *);
|
||||
int sqlite3PutVarint(unsigned char *, u64);
|
||||
int sqlite3GetVarint(const unsigned char *, u64 *);
|
||||
int sqlite3GetVarint32(const unsigned char *, u32 *);
|
||||
|
||||
136
src/utf.c
136
src/utf.c
@ -12,7 +12,7 @@
|
||||
** This file contains routines used to translate between UTF-8,
|
||||
** UTF-16, UTF-16BE, and UTF-16LE.
|
||||
**
|
||||
** $Id: utf.c,v 1.46 2007/05/10 17:23:12 drh Exp $
|
||||
** $Id: utf.c,v 1.47 2007/05/15 11:55:09 drh Exp $
|
||||
**
|
||||
** Notes on UTF-8:
|
||||
**
|
||||
@ -34,29 +34,6 @@
|
||||
** 0xff 0xfe little-endian utf-16 follows
|
||||
** 0xfe 0xff big-endian utf-16 follows
|
||||
**
|
||||
**
|
||||
** Handling of malformed strings:
|
||||
**
|
||||
** SQLite accepts and processes malformed strings without an error wherever
|
||||
** possible. However this is not possible when converting between UTF-8 and
|
||||
** UTF-16.
|
||||
**
|
||||
** When converting malformed UTF-8 strings to UTF-16, one instance of the
|
||||
** replacement character U+FFFD for each byte that cannot be interpeted as
|
||||
** part of a valid unicode character.
|
||||
**
|
||||
** When converting malformed UTF-16 strings to UTF-8, one instance of the
|
||||
** replacement character U+FFFD for each pair of bytes that cannot be
|
||||
** interpeted as part of a valid unicode character.
|
||||
**
|
||||
** This file contains the following public routines:
|
||||
**
|
||||
** sqlite3VdbeMemTranslate() - Translate the encoding used by a Mem* string.
|
||||
** sqlite3VdbeMemHandleBom() - Handle byte-order-marks in UTF16 Mem* strings.
|
||||
** sqlite3Utf16ByteLen() - Calculate byte-length of a void* UTF16 string.
|
||||
** sqlite3Utf8CharLen() - Calculate char-length of a char* UTF8 string.
|
||||
** sqlite3Utf8LikeCompare() - Do a LIKE match given two UTF8 char* strings.
|
||||
**
|
||||
*/
|
||||
#include "sqliteInt.h"
|
||||
#include <assert.h>
|
||||
@ -69,89 +46,20 @@
|
||||
const int sqlite3one = 1;
|
||||
|
||||
/*
|
||||
** This table maps from the first byte of a UTF-8 character to the number
|
||||
** of trailing bytes expected. A value '4' indicates that the table key
|
||||
** is not a legal first byte for a UTF-8 character.
|
||||
** This lookup table is used to help decode the first byte of
|
||||
** a multi-byte UTF8 character.
|
||||
*/
|
||||
static const u8 xtra_utf8_bytes[256] = {
|
||||
/* 0xxxxxxx */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
||||
/* 10wwwwww */
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
|
||||
/* 110yyyyy */
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
|
||||
/* 1110zzzz */
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
|
||||
/* 11110yyy */
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
const unsigned char sqlite3UtfTrans1[] = {
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
||||
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
|
||||
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
|
||||
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
|
||||
};
|
||||
|
||||
/*
|
||||
** This table maps from the number of trailing bytes in a UTF-8 character
|
||||
** to an integer constant that is effectively calculated for each character
|
||||
** read by a naive implementation of a UTF-8 character reader. The code
|
||||
** in the READ_UTF8 macro explains things best.
|
||||
*/
|
||||
static const int xtra_utf8_bits[] = {
|
||||
0,
|
||||
12416, /* (0xC0 << 6) + (0x80) */
|
||||
925824, /* (0xE0 << 12) + (0x80 << 6) + (0x80) */
|
||||
63447168 /* (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */
|
||||
};
|
||||
|
||||
/*
|
||||
** If a UTF-8 character contains N bytes extra bytes (N bytes follow
|
||||
** the initial byte so that the total character length is N+1) then
|
||||
** masking the character with utf8_mask[N] must produce a non-zero
|
||||
** result. Otherwise, we have an (illegal) overlong encoding.
|
||||
*/
|
||||
static const int utf_mask[] = {
|
||||
0x00000000,
|
||||
0xffffff80,
|
||||
0xfffff800,
|
||||
0xffff0000,
|
||||
};
|
||||
|
||||
#define READ_UTF8(zIn, c) { \
|
||||
int xtra; \
|
||||
c = *(zIn)++; \
|
||||
xtra = xtra_utf8_bytes[c]; \
|
||||
switch( xtra ){ \
|
||||
case 4: c = (int)0xFFFD; break; \
|
||||
case 3: c = (c<<6) + *(zIn)++; \
|
||||
case 2: c = (c<<6) + *(zIn)++; \
|
||||
case 1: c = (c<<6) + *(zIn)++; \
|
||||
c -= xtra_utf8_bits[xtra]; \
|
||||
if( (utf_mask[xtra]&c)==0 \
|
||||
|| (c&0xFFFFF800)==0xD800 \
|
||||
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
|
||||
} \
|
||||
}
|
||||
int sqlite3ReadUtf8(const unsigned char *z){
|
||||
int c;
|
||||
READ_UTF8(z, c);
|
||||
return c;
|
||||
}
|
||||
|
||||
#define SKIP_UTF8(zIn) { \
|
||||
zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1); \
|
||||
}
|
||||
|
||||
#define WRITE_UTF8(zOut, c) { \
|
||||
if( c<0x00080 ){ \
|
||||
*zOut++ = (c&0xFF); \
|
||||
@ -346,14 +254,14 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
|
||||
if( desiredEnc==SQLITE_UTF16LE ){
|
||||
/* UTF-8 -> UTF-16 Little-endian */
|
||||
while( zIn<zTerm ){
|
||||
READ_UTF8(zIn, c);
|
||||
SQLITE_READ_UTF8(zIn, c);
|
||||
WRITE_UTF16LE(z, c);
|
||||
}
|
||||
}else{
|
||||
assert( desiredEnc==SQLITE_UTF16BE );
|
||||
/* UTF-8 -> UTF-16 Big-endian */
|
||||
while( zIn<zTerm ){
|
||||
READ_UTF8(zIn, c);
|
||||
SQLITE_READ_UTF8(zIn, c);
|
||||
WRITE_UTF16BE(z, c);
|
||||
}
|
||||
}
|
||||
@ -457,17 +365,18 @@ int sqlite3VdbeMemHandleBom(Mem *pMem){
|
||||
** number of unicode characters in the first nByte of pZ (or up to
|
||||
** the first 0x00, whichever comes first).
|
||||
*/
|
||||
int sqlite3Utf8CharLen(const char *z, int nByte){
|
||||
int sqlite3Utf8CharLen(const char *zIn, int nByte){
|
||||
int r = 0;
|
||||
const char *zTerm;
|
||||
const u8 *z = (const u8*)zIn;
|
||||
const u8 *zTerm;
|
||||
if( nByte>=0 ){
|
||||
zTerm = &z[nByte];
|
||||
}else{
|
||||
zTerm = (const char *)(-1);
|
||||
zTerm = (const u8*)(-1);
|
||||
}
|
||||
assert( z<=zTerm );
|
||||
while( *z!=0 && z<zTerm ){
|
||||
SKIP_UTF8(z);
|
||||
SQLITE_SKIP_UTF8(z);
|
||||
r++;
|
||||
}
|
||||
return r;
|
||||
@ -589,7 +498,7 @@ int sqlite3Utf8To8(unsigned char *zIn){
|
||||
int c;
|
||||
|
||||
while(1){
|
||||
READ_UTF8(zIn, c);
|
||||
SQLITE_READ_UTF8(zIn, c);
|
||||
if( c==0 ) break;
|
||||
if( c!=0xfffd ){
|
||||
WRITE_UTF8(zOut, c);
|
||||
@ -617,8 +526,9 @@ void sqlite3UtfSelfTest(){
|
||||
z = zBuf;
|
||||
WRITE_UTF8(z, i);
|
||||
n = z-zBuf;
|
||||
z[0] = 0;
|
||||
z = zBuf;
|
||||
READ_UTF8(z, c);
|
||||
SQLITE_READ_UTF8(z, c);
|
||||
t = i;
|
||||
if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD;
|
||||
if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD;
|
||||
@ -630,6 +540,7 @@ void sqlite3UtfSelfTest(){
|
||||
z = zBuf;
|
||||
WRITE_UTF16LE(z, i);
|
||||
n = z-zBuf;
|
||||
z[0] = 0;
|
||||
z = zBuf;
|
||||
READ_UTF16LE(z, c);
|
||||
assert( c==i );
|
||||
@ -640,6 +551,7 @@ void sqlite3UtfSelfTest(){
|
||||
z = zBuf;
|
||||
WRITE_UTF16BE(z, i);
|
||||
n = z-zBuf;
|
||||
z[0] = 0;
|
||||
z = zBuf;
|
||||
READ_UTF16BE(z, c);
|
||||
assert( c==i );
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user