mirrors
/
sqlite
mirror of https://github.com/sqlite/sqlite
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimisation for unicode encoding conversion routines. (CVS 1614) 

FossilOrigin-Name: 39a415eaa65964742e40b7ea4d471fa04007c6c9
This commit is contained in:
danielk1977 2004-06-18 04:24:54 +00:00
parent a285422922
commit bfd6cce56b
15 changed files with 674 additions and 1027 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
manifest
View File

@ -1,5 +1,5 @@
C Documentation\supdates\sin\spreparation\sfor\sthe\srelease\sof\sversion\s3.0.0.\s(CVS\s1613)
D 2004-06-17T19:04:17
C Optimisation\sfor\sunicode\sencoding\sconversion\sroutines.\s(CVS\s1614)
D 2004-06-18T04:24:54
F Makefile.in ab7b0d5118e2da97bac66be8684a1034e3500f5a
F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@ -28,7 +28,7 @@ F src/attach.c 93b8ecec4a8d7b4e9f2479e2327d90c9d01765e8
F src/auth.c 204e1e9c45e64315589bc8b62cba5d9de29b6a3c
F src/btree.c 0cf8a52a57a7eb13d50719114ee1fa353e89d7d3
F src/btree.h 32f96abef464cf8765b23ca669acfe90d191fcc5
F src/build.c ed0bff93bf8cbcf853f0ddf0a99c8e5fd02d0db0
F src/build.c f3f7634d21ae3d048c62bea98f99b2b4d2c0ae2b
F src/date.c 65b483caeb0e4dd663667d2f927caa058168ebff
F src/delete.c a5191011b7429dff939df631b8bdcc1714b8d7aa
F src/encode.c a876af473d1d636faa3dca51c7571f2e007eea37
@ -38,7 +38,7 @@ F src/hash.c 440c2f8cb373ee1b4e13a0988489c7cd95d55b6f
F src/hash.h 762d95f1e567664d1eafc1687de755626be962fb
F src/insert.c 1428887f4a7515a7d34e82aaeb76297c79ba378b
F src/legacy.c ad23746f15f67e34577621b1875f639c94839e1f
F src/main.c 63c30e17cae13cbe9ccc037507cd03a27fc3d67b
F src/main.c a7cee3b75b74aec795e26175cd3070fe0909b297
F src/md5.c d77a389955759c8329bb357e3d71bac3d6eb710b
F src/os.h 1cb5f0293a30288451fe3c0c73815cf208212ed1
F src/os_common.h ba1b7306e16e2091718f2c48db0fe6c1d7a31bb8
@ -57,26 +57,26 @@ F src/random.c eff68e3f257e05e81eae6c4d50a51eb88beb4ff3
F src/select.c 91570af7f13d1c5584f3b711676e1226ee12194e
F src/shell.c ca519519dcbbc582f6d88f7d0e7583b857fd3469
F src/sqlite.h.in eacd23af3280603cd8b48979c6f9297b3ac2935c
F src/sqliteInt.h 59c1e1b637e7cf329f39425469bc4dbeab1fa1d5
F src/sqliteInt.h 7d8145b6828826683f44b8c40930c34c8ec1f3c3
F src/table.c af14284fa36c8d41f6829e3f2819dce07d3e2de2
F src/tclsqlite.c 6f88be282a3617c6739c043fd13571b5f2fe9922
F src/test1.c 49c69d360ce09f8ecf56ec5e55fba625703d8b21
F src/test1.c 963f4417d1770f5238672ff5b817a9103842eeb7
F src/test2.c db0a0c30b59f1639cb8c2572be06439402833cc8
F src/test3.c 7247090d15a5a43823079b6fd8dad1ed3cccdedf
F src/test4.c a921a69821fd30209589228e64f94e9f715b6fe2
F src/test5.c 13ac1db35b03acbc197ceb245fe862da5ebf2369
F src/tokenize.c 183c5d7da11affab5d70d903d33409c8c0ce6c5b
F src/test5.c 1b7c275b2929dbd034a567255d2aca339410d1d6
F src/tokenize.c ec8d990bd297c68998cf7b4109b2832ec8b6f8ee
F src/trigger.c 3325207ea41d2d429e70370ce2ff987fd49b7ada
F src/update.c f9a03233577e0c3d57234d1957963875fc941da2
F src/utf.c 92f0dd69bd1c1e4eff93c6038dd91b37ac1b9a65
F src/util.c 6e93dad9a17b34f37fc270ba871b224240168bf0
F src/utf.c 7464315fd4b627b96801444f3671f3e9930879c6
F src/util.c 9d9ef854e9a8744a7ebaee02ee26eb0aaa8a9987
F src/vacuum.c f9561c8095407a970af4e6a304b77c4083433d3e
F src/vdbe.c 22dcf0e98c6316d0a8328d828cce4adc5b8668bb
F src/vdbe.c 2a0e908e46ea6822f215cb0f35ec31f5c2832629
F src/vdbe.h 46f74444a213129bc4b5ce40124dd8ed613b0cde
F src/vdbeInt.h 4e636b1b6c18d1d85b085fe0e5a19d45ad85f382
F src/vdbeapi.c ee350b552fc4c1c695b760f914f69e9c5556e829
F src/vdbeInt.h 96a6e88b82386f2741f0addbe00a8bf38f8e8bfc
F src/vdbeapi.c 229d40ead855cbf154eaf4ee5e5f3d7463100003
F src/vdbeaux.c dc0e7d3bdf3b6f322448b4bee29fe5bec656b4d4
F src/vdbemem.c c00e939cfb19380fc80349e143b541a45ab53424
F src/vdbemem.c 9359c53386e070fea9f5403cab0c6f0cfe36496b
F src/where.c 6507074d8ce3f78e7a4cd33f667f11e62020553e
F test/all.test 569a92a8ee88f5300c057cc4a8f50fbbc69a3242
F test/attach.test aed659e52635662bcd5069599aaca823533edf5a
@ -105,7 +105,7 @@ F test/conflict.test 45ce1e44ea748944aed233df8c278a9e1c4c87cc
F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
F test/date.test aed5030482ebc02bd8d386c6c86a29f694ab068d
F test/delete.test ac14bd6df5f1581d646eebc013d6b844a885dcf6
F test/enc.test a55481d45ff493804e8d88357feb4642fc50a6b2
F test/enc.test dfdde0f660b592acbdafd107d15b2b7dd2a54f45
F test/enc2.test 8b00df318822731ea545365629bc9f52d73a3c6f
F test/expr.test 2262ade5e4e98e359b11e40b01cb3498ac7fa42b
F test/fkey1.test d65c824459916249bee501532d6154ddab0b5db7
@ -225,7 +225,7 @@ F www/tclsqlite.tcl 19191cf2a1010eaeff74c51d83fd5f5a4d899075
F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
F www/version3.tcl f9a4c1a12864e129f6717d22313be01d5657e3aa
F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4
P a71a9ff114f2162696d8d37fbc87b315568f6dcb
R 0c1c395d17282923e97a25ac9186d69a
U drh
Z 95db50b2b10478702338303008862d5b
P 9fb29f7331254b601b71f0fe7d77f91915272dc7
R 687bc7a5f25dd3652f072255ac9e7494
U danielk1977
Z bf8defe1dd292c16a3a6748fcdae004f

2
manifest.uuid
View File

@ -1 +1 @@
9fb29f7331254b601b71f0fe7d77f91915272dc7
39a415eaa65964742e40b7ea4d471fa04007c6c9

23
src/build.c
View File

@ -23,7 +23,7 @@
** ROLLBACK
** PRAGMA
**
** $Id: build.c,v 1.220 2004/06/17 06:13:34 danielk1977 Exp $
** $Id: build.c,v 1.221 2004/06/18 04:24:54 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include <ctype.h>
@ -959,15 +959,12 @@ static void callCollNeeded(sqlite *db, const char *zName, int nName){
db->xCollNeeded(db->pCollNeededArg, db, (int)db->enc, zExternal);
}
if( db->xCollNeeded16 ){
if( SQLITE_BIGENDIAN ){
zExternal = sqlite3utf8to16be(zName, nName);
}else{
zExternal = sqlite3utf8to16le(zName, nName);
}
sqlite3_value *pTmp = sqlite3GetTransientValue(db);
sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF8, SQLITE_STATIC);
zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE);
if( !zExternal ) return;
db->xCollNeeded16(db->pCollNeededArg, db, (int)db->enc, zExternal);
}
if( zExternal ) sqliteFree(zExternal);
}
static int synthCollSeq(Parse *pParse, CollSeq *pColl){
@ -2627,3 +2624,15 @@ void sqlite3EndWriteOperation(Parse *pParse){
/* Delete me! */
return;
}
/*
** Return the transient sqlite3_value object used for encoding conversions
** during SQL compilation.
*/
sqlite3_value *sqlite3GetTransientValue(sqlite *db){
if( !db->pValue ){
db->pValue = sqlite3ValueNew();
}
return db->pValue;
}

125
src/main.c
View File

@ -14,7 +14,7 @@
** other files are for internal use by SQLite and should not be
** accessed by users of the library.
**
** $Id: main.c,v 1.224 2004/06/16 12:00:56 danielk1977 Exp $
** $Id: main.c,v 1.225 2004/06/18 04:24:54 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
@ -515,6 +515,12 @@ void sqlite3_close(sqlite *db){
sqlite3HashClear(&db->aFunc);
sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */
if( db->pValue ){
sqlite3ValueFree(db->pValue);
}
if( db->pErr ){
sqlite3ValueFree(db->pErr);
}
sqliteFree(db);
}
@ -749,14 +755,17 @@ int sqlite3_create_function16(
void (*xFinal)(sqlite3_context*)
){
int rc;
char *zFunctionName8;
zFunctionName8 = sqlite3utf16to8(zFunctionName, -1, SQLITE_BIGENDIAN);
if( !zFunctionName8 ){
char const *zFunc8;
sqlite3_value *pTmp = sqlite3GetTransientValue(db);
sqlite3ValueSetStr(pTmp, -1, zFunctionName, SQLITE_UTF16NATIVE,SQLITE_STATIC);
zFunc8 = sqlite3ValueText(pTmp, SQLITE_UTF8);
if( !zFunc8 ){
return SQLITE_NOMEM;
}
rc = sqlite3_create_function(db, zFunctionName8, nArg, eTextRep,
rc = sqlite3_create_function(db, zFunc8, nArg, eTextRep,
iCollateArg, pUserData, xFunc, xStep, xFinal);
sqliteFree(zFunctionName8);
return rc;
}
@ -844,16 +853,16 @@ int sqlite3BtreeFactory(
** error.
*/
const char *sqlite3_errmsg(sqlite3 *db){
if( !db ){
if( !db || !db->pErr ){
/* If db is NULL, then assume that a malloc() failed during an
** sqlite3_open() call.
*/
return sqlite3ErrStr(SQLITE_NOMEM);
}
if( db->zErrMsg ){
return db->zErrMsg;
if( !sqlite3_value_text(db->pErr) ){
return sqlite3ErrStr(db->errCode);
}
return sqlite3ErrStr(db->errCode);
return sqlite3_value_text(db->pErr);
}
/*
@ -861,38 +870,32 @@ const char *sqlite3_errmsg(sqlite3 *db){
** error.
*/
const void *sqlite3_errmsg16(sqlite3 *db){
if( !db ){
/* If db is NULL, then assume that a malloc() failed during an
** sqlite3_open() call. We have a static version of the string
** "out of memory" encoded using UTF-16 just for this purpose.
**
** Because all the characters in the string are in the unicode
** range 0x00-0xFF, if we pad the big-endian string with a
** zero byte, we can obtain the little-endian string with
** &big_endian[1].
*/
static char outOfMemBe[] = {
0, 'o', 0, 'u', 0, 't', 0, ' ',
0, 'o', 0, 'f', 0, ' ',
0, 'm', 0, 'e', 0, 'm', 0, 'o', 0, 'r', 0, 'y', 0, 0, 0
};
static char *outOfMemLe = &outOfMemBe[1];
/* Because all the characters in the string are in the unicode
** range 0x00-0xFF, if we pad the big-endian string with a
** zero byte, we can obtain the little-endian string with
** &big_endian[1].
*/
static char outOfMemBe[] = {
0, 'o', 0, 'u', 0, 't', 0, ' ',
0, 'o', 0, 'f', 0, ' ',
0, 'm', 0, 'e', 0, 'm', 0, 'o', 0, 'r', 0, 'y', 0, 0, 0
};
if( SQLITE_BIGENDIAN ){
return (void *)outOfMemBe;
}else{
return (void *)outOfMemLe;
if( db && db->pErr ){
if( !sqlite3_value_text16(db->pErr) ){
sqlite3ValueSetStr(db->pErr, -1, sqlite3ErrStr(db->errCode),
SQLITE_UTF8, SQLITE_STATIC);
}
}
if( !db->zErrMsg16 ){
char const *zErr8 = sqlite3_errmsg(db);
if( SQLITE_BIGENDIAN ){
db->zErrMsg16 = sqlite3utf8to16be(zErr8, -1);
}else{
db->zErrMsg16 = sqlite3utf8to16le(zErr8, -1);
if( sqlite3_value_text16(db->pErr) ){
return sqlite3_value_text16(db->pErr);
}
}
return db->zErrMsg16;
}
/* If db is NULL, then assume that a malloc() failed during an
** sqlite3_open() call. We have a static version of the string
** "out of memory" encoded using UTF-16 just for this purpose.
*/
return (void *)(&outOfMemBe[SQLITE_UTF16NATIVE==SQLITE_UTF16LE?1:0]);
}
int sqlite3_errcode(sqlite3 *db){
@ -1047,11 +1050,14 @@ int sqlite3_prepare16(
** encoded string to UTF-8, then invoking sqlite3_prepare(). The
** tricky bit is figuring out the pointer to return in *pzTail.
*/
char *zSql8 = 0;
char const *zSql8 = 0;
char const *zTail8 = 0;
int rc;
sqlite3_value *pTmp;
zSql8 = sqlite3utf16to8(zSql, nBytes, SQLITE_BIGENDIAN);
pTmp = sqlite3GetTransientValue(db);
sqlite3ValueSetStr(pTmp, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC);
zSql8 = sqlite3ValueText(pTmp, SQLITE_UTF8);
if( !zSql8 ){
sqlite3Error(db, SQLITE_NOMEM, 0);
return SQLITE_NOMEM;
@ -1067,7 +1073,6 @@ int sqlite3_prepare16(
int chars_parsed = sqlite3utf8CharLen(zSql8, zTail8-zSql8);
*pzTail = (u8 *)zSql + sqlite3utf16ByteLen(zSql, chars_parsed);
}
sqliteFree(zSql8);
return rc;
}
@ -1134,7 +1139,6 @@ static int openDatabase(
}
rc = sqlite3BtreeFactory(db, zFilename, 0, MAX_PAGES, &db->aDb[0].pBt);
if( rc!=SQLITE_OK ){
/* FIX ME: sqlite3BtreeFactory() should call sqlite3Error(). */
sqlite3Error(db, rc, 0);
db->magic = SQLITE_MAGIC_CLOSED;
goto opendb_out;
@ -1148,6 +1152,7 @@ static int openDatabase(
*/
sqlite3RegisterBuiltinFunctions(db);
if( rc==SQLITE_OK ){
sqlite3Error(db, SQLITE_OK, 0);
db->magic = SQLITE_MAGIC_OPEN;
}else{
sqlite3Error(db, rc, "%s", zErrMsg, 0);
@ -1177,21 +1182,24 @@ int sqlite3_open16(
const void *zFilename,
sqlite3 **ppDb
){
char *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */
int rc;
char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */
int rc = SQLITE_NOMEM;
sqlite3_value *pVal;
assert( ppDb );
zFilename8 = sqlite3utf16to8(zFilename, -1, SQLITE_BIGENDIAN);
if( !zFilename8 ){
*ppDb = 0;
return SQLITE_NOMEM;
*ppDb = 0;
pVal = sqlite3ValueNew();
sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC);
zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8);
if( zFilename8 ){
rc = openDatabase(zFilename8, ppDb);
if( rc==SQLITE_OK && *ppDb ){
sqlite3_exec(*ppDb, "PRAGMA encoding = 'UTF-16'", 0, 0, 0);
}
}
rc = openDatabase(zFilename8, ppDb);
if( rc==SQLITE_OK && *ppDb ){
sqlite3_exec(*ppDb, "PRAGMA encoding = 'UTF-16'", 0, 0, 0);
if( pVal ){
sqlite3ValueFree(pVal);
}
sqliteFree(zFilename8);
return rc;
}
@ -1273,10 +1281,11 @@ int sqlite3_create_collation16(
int(*xCompare)(void*,int,const void*,int,const void*)
){
int rc;
char *zName8 = sqlite3utf16to8(zName, -1, SQLITE_BIGENDIAN);
rc = sqlite3_create_collation(db, zName8, enc, pCtx, xCompare);
sqliteFree(zName8);
return rc;
char const *zName8;
sqlite3_value *pTmp = sqlite3GetTransientValue(db);
sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF16NATIVE, SQLITE_STATIC);
zName8 = sqlite3ValueText(pTmp, SQLITE_UTF8);
return sqlite3_create_collation(db, zName8, enc, pCtx, xCompare);
}
/*

21
src/sqliteInt.h
View File

@ -11,7 +11,7 @@
*************************************************************************
** Internal interface definitions for SQLite.
**
** @(#) $Id: sqliteInt.h,v 1.286 2004/06/17 05:36:44 danielk1977 Exp $
** @(#) $Id: sqliteInt.h,v 1.287 2004/06/18 04:24:54 danielk1977 Exp $
*/
#include "config.h"
#include "sqlite3.h"
@ -194,6 +194,7 @@ extern const int sqlite3one;
# define sqliteStrNDup(X,Y) sqlite3StrNDup_(X,Y,__FILE__,__LINE__)
void sqlite3StrRealloc(char**);
#else
# define sqlite3FreeX sqliteFree
# define sqlite3Realloc_(X,Y) sqliteRealloc(X,Y)
# define sqlite3StrRealloc(X)
#endif
@ -422,14 +423,17 @@ struct sqlite {
#endif
int errCode; /* Most recent error code (SQLITE_*) */
char *zErrMsg; /* Most recent error message (UTF-8 encoded) */
void *zErrMsg16; /* Most recent error message (UTF-16 encoded) */
u8 enc; /* Text encoding for this database. */
u8 autoCommit; /* The auto-commit flag. */
int nMaster; /* Length of master journal name. -1=unknown */
void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*);
void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*);
void *pCollNeededArg;
sqlite3_value *pValue; /* Value used for transient conversions */
sqlite3_value *pErr; /* Most recent error message */
char *zErrMsg; /* Most recent error message (UTF-8 encoded) */
char *zErrMsg16; /* Most recent error message (UTF-8 encoded) */
};
/*
@ -1213,6 +1217,7 @@ void sqlite3RealToSortable(double r, char *);
char *sqlite3StrDup_(const char*,char*,int);
char *sqlite3StrNDup_(const char*, int,char*,int);
void sqlite3CheckMemory(void*,int);
void sqlite3FreeX(void *p);
#else
void *sqliteMalloc(int);
void *sqliteMallocRaw(int);
@ -1375,11 +1380,6 @@ char *sqlite3_snprintf(int,char*,const char*,...);
int sqlite3GetInt32(const char *, int*);
int sqlite3GetInt64(const char *, i64*);
int sqlite3FitsIn64Bits(const char *);
unsigned char *sqlite3utf16to8(const void *pData, int N, int big_endian);
void *sqlite3utf8to16be(const unsigned char *pIn, int N);
void *sqlite3utf8to16le(const unsigned char *pIn, int N);
void sqlite3utf16to16le(void *pData, int N);
void sqlite3utf16to16be(void *pData, int N);
int sqlite3utf16ByteLen(const void *pData, int nChar);
int sqlite3utf8CharLen(const char *pData, int nByte);
int sqlite3utf8LikeCompare(const unsigned char*, const unsigned char*);
@ -1396,8 +1396,6 @@ int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity);
char sqlite3ExprAffinity(Expr *pExpr);
int sqlite3atoi64(const char*, i64*);
void sqlite3Error(sqlite *, int, const char*,...);
int sqlite3utfTranslate(const void *, int , u8 , void **, int *, u8);
u8 sqlite3UtfReadBom(const void *zData, int nData);
void *sqlite3HexToBlob(const char *z);
int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
const char *sqlite3ErrStr(int);
@ -1412,6 +1410,7 @@ int sqlite3CheckObjectName(Parse *, const char *);
const void *sqlite3ValueText(sqlite3_value*, u8);
int sqlite3ValueBytes(sqlite3_value*, u8);
void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8);
void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*));
void sqlite3ValueFree(sqlite3_value*);
sqlite3_value *sqlite3ValueNew();
sqlite3_value *sqlite3GetTransientValue(sqlite *db);

6
src/test1.c
View File

@ -13,7 +13,7 @@
** is not included in the SQLite library. It is used for automated
** testing of the SQLite library.
**
** $Id: test1.c,v 1.77 2004/06/15 02:44:19 danielk1977 Exp $
** $Id: test1.c,v 1.78 2004/06/18 04:24:55 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "tcl.h"
@ -940,9 +940,9 @@ static int test_collate_func(
}
pVal = sqlite3ValueNew();
sqlite3ValueSetStr(pVal, nA, zA, encin);
sqlite3ValueSetStr(pVal, nA, zA, encin, SQLITE_STATIC);
Tcl_ListObjAppendElement(i,pX,Tcl_NewStringObj(sqlite3_value_text(pVal),-1));
sqlite3ValueSetStr(pVal, nB, zB, encin);
sqlite3ValueSetStr(pVal, nB, zB, encin, SQLITE_STATIC);
Tcl_ListObjAppendElement(i,pX,Tcl_NewStringObj(sqlite3_value_text(pVal),-1));
sqlite3ValueFree(pVal);

284
src/test5.c
View File

@ -15,7 +15,7 @@
** is used for testing the SQLite routines for converting between
** the various supported unicode encodings.
**
** $Id: test5.c,v 1.10 2004/06/12 00:42:35 danielk1977 Exp $
** $Id: test5.c,v 1.11 2004/06/18 04:24:55 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "vdbeInt.h"
@ -24,195 +24,6 @@
#include <stdlib.h>
#include <string.h>
/*
** Return the number of bytes up to and including the first pair of
** 0x00 bytes in *pStr.
*/
static int utf16_length(const unsigned char *pZ){
const unsigned char *pC1 = pZ;
const unsigned char *pC2 = pZ+1;
while( *pC1 || *pC2 ){
pC1 += 2;
pC2 += 2;
}
return (pC1-pZ)+2;
}
/*
** tclcmd: sqlite_utf8to16le STRING
** title: Convert STRING from utf-8 to utf-16le
**
** Return the utf-16le encoded string
*/
static int sqlite_utf8to16le(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
unsigned char *out;
unsigned char *in;
Tcl_Obj *res;
if( objc!=2 ){
Tcl_AppendResult(interp, "wrong # args: should be \"",
Tcl_GetStringFromObj(objv[0], 0), "<utf-8 encoded-string>", 0);
return TCL_ERROR;
}
in = Tcl_GetString(objv[1]);
out = (unsigned char *)sqlite3utf8to16le(in, -1);
res = Tcl_NewByteArrayObj(out, utf16_length(out));
sqliteFree(out);
Tcl_SetObjResult(interp, res);
return TCL_OK;
}
/*
** tclcmd: sqlite_utf8to16be STRING
** title: Convert STRING from utf-8 to utf-16be
**
** Return the utf-16be encoded string
*/
static int sqlite_utf8to16be(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
unsigned char *out;
unsigned char *in;
Tcl_Obj *res;
if( objc!=2 ){
Tcl_AppendResult(interp, "wrong # args: should be \"",
Tcl_GetStringFromObj(objv[0], 0), "<utf-8 encoded-string>", 0);
return TCL_ERROR;
}
in = Tcl_GetByteArrayFromObj(objv[1], 0);
in = Tcl_GetString(objv[1]);
out = (unsigned char *)sqlite3utf8to16be(in, -1);
res = Tcl_NewByteArrayObj(out, utf16_length(out));
sqliteFree(out);
Tcl_SetObjResult(interp, res);
return TCL_OK;
}
/*
** tclcmd: sqlite_utf16to16le STRING
** title: Convert STRING from utf-16 in native byte order to utf-16le
**
** Return the utf-16le encoded string. If the input string contains
** a byte-order mark, then the byte order mark should override the
** native byte order.
*/
static int sqlite_utf16to16le(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
unsigned char *out;
unsigned char *in;
int in_len;
Tcl_Obj *res;
if( objc!=2 ){
Tcl_AppendResult(interp, "wrong # args: should be \"",
Tcl_GetStringFromObj(objv[0], 0), "<utf-16 encoded-string>", 0);
return TCL_ERROR;
}
in = Tcl_GetByteArrayFromObj(objv[1], &in_len);
out = (unsigned char *)sqliteMalloc(in_len);
memcpy(out, in, in_len);
sqlite3utf16to16le(out, -1);
res = Tcl_NewByteArrayObj(out, utf16_length(out));
sqliteFree(out);
Tcl_SetObjResult(interp, res);
return TCL_OK;
}
/*
** tclcmd: sqlite_utf16to16be STRING
** title: Convert STRING from utf-16 in native byte order to utf-16be
**
** Return the utf-16be encoded string. If the input string contains
** a byte-order mark, then the byte order mark should override the
** native byte order.
*/
static int sqlite_utf16to16be(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
unsigned char *out;
unsigned char *in;
int in_len;
Tcl_Obj *res;
if( objc!=2 ){
Tcl_AppendResult(interp, "wrong # args: should be \"",
Tcl_GetStringFromObj(objv[0], 0), "<utf-16 encoded-string>", 0);
return TCL_ERROR;
}
in = Tcl_GetByteArrayFromObj(objv[1], &in_len);
out = (unsigned char *)sqliteMalloc(in_len);
memcpy(out, in, in_len);
sqlite3utf16to16be(out, -1);
res = Tcl_NewByteArrayObj(out, utf16_length(out));
sqliteFree(out);
Tcl_SetObjResult(interp, res);
return TCL_OK;
}
/*
** tclcmd: sqlite_utf16to8 STRING
** title: Convert STRING from utf-16 in native byte order to utf-8
**
** Return the utf-8 encoded string. If the input string contains
** a byte-order mark, then the byte order mark should override the
** native byte order.
*/
static int sqlite_utf16to8(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
unsigned char *out;
unsigned char *in;
Tcl_Obj *res;
if( objc!=2 ){
Tcl_AppendResult(interp, "wrong # args: should be \"",
Tcl_GetStringFromObj(objv[0], 0), " <utf-16 encoded-string>", 0);
return TCL_ERROR;
}
in = Tcl_GetByteArrayFromObj(objv[1], 0);
out = sqlite3utf16to8(in, -1, SQLITE_BIGENDIAN);
res = Tcl_NewByteArrayObj(out, strlen(out)+1);
sqliteFree(out);
Tcl_SetObjResult(interp, res);
return TCL_OK;
}
/*
** The first argument is a TCL UTF-8 string. Return the byte array
** object with the encoded representation of the string, including
@ -281,6 +92,92 @@ static int test_value_overhead(
return TCL_OK;
}
static u8 name_to_enc(Tcl_Interp *interp, Tcl_Obj *pObj){
struct EncName {
char *zName;
u8 enc;
} encnames[] = {
{ "UTF8", SQLITE_UTF8 },
{ "UTF16LE", SQLITE_UTF16LE },
{ "UTF16BE", SQLITE_UTF16BE },
{ "UTF16", SQLITE_UTF16NATIVE },
{ 0, 0 }
};
struct EncName *pEnc;
char *z = Tcl_GetString(pObj);
for(pEnc=&encnames[0]; pEnc->zName; pEnc++){
if( 0==sqlite3StrICmp(z, pEnc->zName) ){
break;
}
}
if( !pEnc->enc ){
Tcl_AppendResult(interp, "No such encoding: ", z, 0);
}
return pEnc->enc;
}
static int test_translate(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
u8 enc_from;
u8 enc_to;
sqlite3_value *pVal;
const char *z;
int len;
if( objc!=4 ){
Tcl_AppendResult(interp, "wrong # args: should be \"",
Tcl_GetStringFromObj(objv[0], 0),
" <string/blob> <from enc> <to enc>", 0
);
return TCL_ERROR;
}
enc_from = name_to_enc(interp, objv[2]);
if( !enc_from ) return TCL_ERROR;
enc_to = name_to_enc(interp, objv[3]);
if( !enc_to ) return TCL_ERROR;
pVal = sqlite3ValueNew();
if( enc_from==SQLITE_UTF8 ){
z = Tcl_GetString(objv[1]);
sqlite3ValueSetStr(pVal, -1, z, enc_from, SQLITE_STATIC);
}else{
z = Tcl_GetByteArrayFromObj(objv[1], &len);
sqlite3ValueSetStr(pVal, -1, z, enc_from, SQLITE_STATIC);
}
z = sqlite3ValueText(pVal, enc_to);
len = sqlite3ValueBytes(pVal, enc_to) + (enc_to==SQLITE_UTF8?1:2);
Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(z, len));
sqlite3ValueFree(pVal);
return TCL_OK;
}
/*
** Usage: translate_selftest
**
** Call sqlite3utfSelfTest() to run the internal tests for unicode
** translation. If there is a problem an assert() will fail.
**/
void sqlite3utfSelfTest();
static int test_translate_selftest(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3utfSelfTest();
return SQLITE_OK;
}
/*
** Register commands with the TCL interpreter.
@ -290,13 +187,10 @@ int Sqlitetest5_Init(Tcl_Interp *interp){
char *zName;
Tcl_ObjCmdProc *xProc;
} aCmd[] = {
{ "sqlite_utf16to8", (Tcl_ObjCmdProc*)sqlite_utf16to8 },
{ "sqlite_utf8to16le", (Tcl_ObjCmdProc*)sqlite_utf8to16le },
{ "sqlite_utf8to16be", (Tcl_ObjCmdProc*)sqlite_utf8to16be },
{ "sqlite_utf16to16le", (Tcl_ObjCmdProc*)sqlite_utf16to16le },
{ "sqlite_utf16to16be", (Tcl_ObjCmdProc*)sqlite_utf16to16be },
{ "binarize", (Tcl_ObjCmdProc*)binarize },
{ "test_value_overhead", (Tcl_ObjCmdProc*)test_value_overhead },
{ "test_translate", (Tcl_ObjCmdProc*)test_translate },
{ "translate_selftest", (Tcl_ObjCmdProc*)test_translate_selftest},
};
int i;
for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){

20
src/tokenize.c
View File

@ -15,7 +15,7 @@
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
** $Id: tokenize.c,v 1.76 2004/05/31 23:56:43 danielk1977 Exp $
** $Id: tokenize.c,v 1.77 2004/06/18 04:24:55 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
@ -701,10 +701,18 @@ int sqlite3_complete(const char *zSql){
** UTF-8.
*/
int sqlite3_complete16(const void *zSql){
int rc;
char *zSql8 = sqlite3utf16to8(zSql, -1, SQLITE_BIGENDIAN);
if( !zSql8 ) return 0;
rc = sqlite3_complete(zSql8);
sqliteFree(zSql8);
sqlite3_value *pVal;
char *zSql8;
int rc = 0;
pVal = sqlite3ValueNew();
sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC);
zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8);
if( zSql8 ){
rc = sqlite3_complete(zSql8);
sqliteFree(zSql8);
}
sqlite3ValueFree(pVal);
return rc;
}

972
src/utf.c
View File

File diff suppressed because it is too large Load Diff

38
src/util.c
View File

@ -14,7 +14,7 @@
** This file contains functions for allocating memory, comparing
** strings, and stuff like that.
**
** $Id: util.c,v 1.102 2004/06/16 07:45:29 danielk1977 Exp $
** $Id: util.c,v 1.103 2004/06/18 04:24:55 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include <stdarg.h>
@ -256,6 +256,13 @@ char *sqlite3StrNDup_(const char *z, int n, char *zFile, int line){
}
return zNew;
}
/*
** A version of sqliteFree that is always a function, not a macro.
*/
void sqlite3FreeX(void *p){
sqliteFree(p);
}
#endif /* SQLITE_DEBUG */
/*
@ -446,23 +453,18 @@ void sqlite3SetNString(char **pz, ...){
** to NULL.
*/
void sqlite3Error(sqlite *db, int err_code, const char *zFormat, ...){
/* Free any existing error message. */
if( db->zErrMsg ){
sqliteFree(db->zErrMsg);
db->zErrMsg = 0;
}
if( db->zErrMsg16 ){
sqliteFree(db->zErrMsg16);
db->zErrMsg16 = 0;
}
/* Set the new error code and error message. */
db->errCode = err_code;
if( zFormat ){
va_list ap;
va_start(ap, zFormat);
db->zErrMsg = sqlite3VMPrintf(zFormat, ap);
va_end(ap);
if( db && (db->pErr || (db->pErr = sqlite3ValueNew())) ){
db->errCode = err_code;
if( zFormat ){
char *z;
va_list ap;
va_start(ap, zFormat);
z = sqlite3VMPrintf(zFormat, ap);
va_end(ap);
sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, sqlite3FreeX);
}else{
sqlite3ValueSetStr(db->pErr, 0, 0, SQLITE_UTF8, SQLITE_STATIC);
}
}
}

39
src/vdbe.c
View File

@ -43,7 +43,7 @@
** in this file for details. If in doubt, do not deviate from existing
** commenting and indentation practices when changing or adding code.
**
** $Id: vdbe.c,v 1.378 2004/06/17 07:53:03 danielk1977 Exp $
** $Id: vdbe.c,v 1.379 2004/06/18 04:24:55 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
@ -361,10 +361,12 @@ static void applyAffinity(Mem *pRec, char affinity, u8 enc){
** Write a nice string representation of the contents of cell pMem
** into buffer zBuf, length nBuf.
*/
void prettyPrintMem(Mem *pMem, char *zBuf, int nBuf){
void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf, int nBuf){
char *zCsr = zBuf;
int f = pMem->flags;
static const char *encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"};
if( f&MEM_Blob ){
int i;
char c;
@ -414,11 +416,6 @@ void prettyPrintMem(Mem *pMem, char *zBuf, int nBuf){
zBuf[k++] = '[';
for(j=0; j<15 && j<pMem->n; j++){
u8 c = pMem->z[j];
/*
if( c==0 && j==pMem->n-1 ) break;
zBuf[k++] = "0123456789ABCDEF"[c>>4];
zBuf[k++] = "0123456789ABCDEF"[c&0xf];
*/
if( c>=0x20 && c<0x7f ){
zBuf[k++] = c;
}else{
@ -426,14 +423,10 @@ void prettyPrintMem(Mem *pMem, char *zBuf, int nBuf){
}
}
zBuf[k++] = ']';
k += sprintf(&zBuf[k], encnames[pMem->enc]);
zBuf[k++] = 0;
}
}
/* Temporary - this is useful in conjunction with prettyPrintMem whilst
** debugging.
*/
char zGdbBuf[100];
#endif
@ -734,20 +727,20 @@ case OP_String8: {
pOp->opcode = OP_String;
if( db->enc!=SQLITE_UTF8 && pOp->p3 ){
char *z = pOp->p3;
if( db->enc==SQLITE_UTF16LE ){
pOp->p3 = sqlite3utf8to16le(z, -1);
}else{
pOp->p3 = sqlite3utf8to16be(z, -1);
}
pTos++;
sqlite3VdbeMemSetStr(pTos, pOp->p3, -1, SQLITE_UTF8, SQLITE_STATIC);
if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pTos, db->enc) ) goto no_mem;
if( SQLITE_OK!=sqlite3VdbeMemDynamicify(pTos) ) goto no_mem;
pTos->flags &= ~(MEM_Dyn);
pTos->flags |= MEM_Static;
if( pOp->p3type==P3_DYNAMIC ){
sqliteFree(z);
sqliteFree(pOp->p3);
}
pOp->p3type = P3_DYNAMIC;
if( !pOp->p3 ) goto no_mem;
pOp->p3 = pTos->z;
break;
}
/* Fall through to the next case, OP_String */
/* Otherwise fall through to the next case, OP_String */
}
/* Opcode: String * * P3
@ -4590,7 +4583,7 @@ default: {
fprintf(p->trace, " r:%g", pTos[i].r);
}else{
char zBuf[100];
prettyPrintMem(&pTos[i], zBuf, 100);
sqlite3VdbeMemPrettyPrint(&pTos[i], zBuf, 100);
fprintf(p->trace, " ");
fprintf(p->trace, zBuf);
}

2
src/vdbeInt.h
View File

@ -390,3 +390,5 @@ void sqlite3VdbeMemRelease(Mem *p);
#ifndef NDEBUG
void sqlite3VdbeMemSanity(Mem*, u8);
#endif
int sqlite3VdbeMemTranslate(Mem*, u8);
void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf, int nBuf);

15
src/vdbeapi.c
View File

@ -518,20 +518,7 @@ int sqlite3_bind_text16(
}
pVar = &p->apVar[i-1];
/* There may or may not be a byte order mark at the start of the UTF-16.
** Either way set 'txt_enc' to the SQLITE_UTF16* value indicating the
** actual byte order used by this string. If the string does happen
** to contain a BOM, then move zData so that it points to the first
** byte after the BOM.
*/
txt_enc = sqlite3UtfReadBom(zData, nData);
if( txt_enc ){
zData = (void *)(((u8 *)zData) + 2);
nData -= 2;
}else{
txt_enc = SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE;
}
rc = sqlite3VdbeMemSetStr(pVar, zData, nData, txt_enc, xDel);
rc = sqlite3VdbeMemSetStr(pVar, zData, nData, SQLITE_UTF16NATIVE, xDel);
if( rc ){
return rc;
}

93
src/vdbemem.c
View File

@ -21,63 +21,23 @@
#include "vdbeInt.h"
/*
** If pMem is a string object, this routine sets the encoding of the string
** (to one of UTF-8 or UTF16) and whether or not the string is
** nul-terminated. If pMem is not a string object, then this routine is
** a no-op.
** If pMem is an object with a valid string representation, this routine
** ensures the internal encoding for the string representation is
** 'desiredEnc', one of SQLITE_UTF8, SQLITE_UTF16LE or SQLITE_UTF16BE.
**
** The second argument, "desiredEnc" is one of TEXT_Utf8, TEXT_Utf16le
** or TEXT_Utf16be. This routine changes the encoding of pMem to match
** desiredEnc.
** If pMem is not a string object, or the encoding of the string
** representation is already stored using the requested encoding, then this
** routine is a no-op.
**
** SQLITE_OK is returned if the conversion is successful (or not required).
** SQLITE_NOMEM may be returned if a malloc() fails during conversion
** between formats.
*/
int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){
/* If this is not a string, or if it is a string but the encoding is
** already correct, do nothing. */
if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){
return SQLITE_OK;
}
if( pMem->enc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF8 ){
/* If the current encoding does not match the desired encoding, then
** we will need to do some translation between encodings.
*/
char *z;
int n;
int rc;
rc = sqlite3utfTranslate(pMem->z, pMem->n, pMem->enc, (void **)&z,
&n, desiredEnc);
if( rc!=SQLITE_OK ){
return rc;
}
sqlite3VdbeMemRelease(pMem);
/* Result of sqlite3utfTranslate is currently always dynamically
** allocated and nul terminated. This might be altered as a performance
** enhancement later.
*/
pMem->z = z;
pMem->n = n;
pMem->flags &= ~(MEM_Ephem | MEM_Short | MEM_Static);
pMem->flags |= MEM_Str | MEM_Dyn | MEM_Term;
pMem->xDel = 0;
}else{
/* Must be translating between UTF-16le and UTF-16be. */
int i;
u8 *pFrom, *pTo;
sqlite3VdbeMemMakeWriteable(pMem);
for(i=0, pFrom=pMem->z, pTo=&pMem->z[1]; i<pMem->n; i+=2, pFrom+=2,pTo+=2){
u8 temp = *pFrom;
*pFrom = *pTo;
*pTo = temp;
}
}
pMem->enc = desiredEnc;
return SQLITE_OK;
return sqlite3VdbeMemTranslate(pMem, desiredEnc);
}
/*
@ -405,16 +365,19 @@ int sqlite3VdbeMemSetStr(
case SQLITE_UTF16LE:
case SQLITE_UTF16BE:
pMem->flags |= MEM_Str;
if( n<0 ){
pMem->n = sqlite3utf16ByteLen(z,-1);
if( pMem->n<0 ){
pMem->n = sqlite3utf16ByteLen(pMem->z,-1);
pMem->flags |= MEM_Term;
}
if( sqlite3VdbeMemHandleBom(pMem) ){
return SQLITE_NOMEM;
}
break;
default:
assert(0);
}
if( xDel==SQLITE_TRANSIENT ){
if( pMem->flags&MEM_Ephem ){
return sqlite3VdbeMemMakeWriteable(pMem);
}
return SQLITE_OK;
@ -498,11 +461,9 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
assert( pMem1->enc==SQLITE_UTF8 ||
pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE );
/* FIX ME: This may fail if the collation sequence is deleted after
** this vdbe program is compiled. We cannot just use BINARY in this
** case as this may lead to a segfault caused by traversing an index
** table incorrectly. We need to return an error to the user in this
** case.
/* This assert may fail if the collation sequence is deleted after this
** vdbe program is compiled. The documentation defines this as an
** undefined condition. A crash is usual result.
*/
assert( !pColl || pColl->xCmp );
@ -645,22 +606,17 @@ void sqlite3VdbeMemSanity(Mem *pMem, u8 db_enc){
** SQLITE_UTF8.
*/
const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
if( !pVal ) return 0;
assert( enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE || enc==SQLITE_UTF8);
if( pVal->flags&MEM_Null ){
/* For a NULL return a NULL Pointer */
return 0;
}
if( pVal->flags&MEM_Str ){
/* If there is already a string representation, make sure it is in
** encoded in the required UTF-16 byte order.
*/
sqlite3VdbeChangeEncoding(pVal, enc);
}else if( !(pVal->flags&MEM_Blob) ){
/* Otherwise, unless this is a blob, convert it to a UTF-16 string */
sqlite3VdbeMemStringify(pVal, enc);
}
return (const void *)(pVal->z);
}
@ -673,12 +629,19 @@ sqlite3_value* sqlite3ValueNew(){
return p;
}
void sqlite3ValueSetStr(sqlite3_value *v, int n, const void *z, u8 enc){
sqlite3VdbeMemSetStr((Mem *)v, z, n, enc, SQLITE_STATIC);
void sqlite3ValueSetStr(
sqlite3_value *v,
int n,
const void *z,
u8 enc,
void (*xDel)(void*)
){
if( v ) sqlite3VdbeMemSetStr((Mem *)v, z, n, enc, xDel);
}
void sqlite3ValueFree(sqlite3_value *v){
sqlite3ValueSetStr(v, 0, 0, SQLITE_UTF8);
if( !v ) return;
sqlite3ValueSetStr(v, 0, 0, SQLITE_UTF8, SQLITE_STATIC);
sqliteFree(v);
}

23
test/enc.test
View File

@ -13,7 +13,7 @@
# various suported unicode encodings (UTF-8, UTF-16, UTF-16le and
# UTF-16be).
#
# $Id: enc.test,v 1.1 2004/05/22 08:16:11 danielk1977 Exp $
# $Id: enc.test,v 1.2 2004/06/18 04:24:56 danielk1977 Exp $
set testdir [file dirname $argv0]
source $testdir/tester.tcl
@ -74,7 +74,7 @@ proc swap_byte_order {utf16} {
proc test_conversion {testname str} {
# Step 1.
set utf16le_sqlite [sqlite_utf8to16le $str]
set utf16le_sqlite [test_translate $str UTF8 UTF16LE]
set utf16le_tcl [encoding convertto unicode $str]
append utf16le_tcl "\x00\x00"
if { $::tcl_platform(byteOrder)!="littleEndian" } {
@ -84,7 +84,7 @@ proc test_conversion {testname str} {
set utf16le $utf16le_tcl
# Step 2.
set utf16be_sqlite [sqlite_utf8to16be $str]
set utf16be_sqlite [test_translate $str UTF8 UTF16BE]
set utf16be_tcl [encoding convertto unicode $str]
append utf16be_tcl "\x00\x00"
if { $::tcl_platform(byteOrder)=="littleEndian" } {
@ -99,40 +99,42 @@ proc test_conversion {testname str} {
} else {
set utf16 $utf16be
}
set utf8_sqlite [sqlite_utf16to8 $utf16]
set utf8_sqlite [test_translate $utf16 UTF16 UTF8]
do_bincmp_test $testname.3 $utf8_sqlite [binarize $str]
# Step 4 (little endian).
append utf16le_bom "\xFF\xFE" $utf16le
set utf8_sqlite [sqlite_utf16to8 $utf16le_bom]
set utf8_sqlite [test_translate $utf16le_bom UTF16 UTF8]
do_bincmp_test $testname.4.le $utf8_sqlite [binarize $str]
# Step 4 (big endian).
append utf16be_bom "\xFE\xFF" $utf16be
set utf8_sqlite [sqlite_utf16to8 $utf16be_bom]
set utf8_sqlite [test_translate $utf16be_bom UTF16 UTF8]
do_bincmp_test $testname.4.be $utf8_sqlite [binarize $str]
# Step 5 (little endian to little endian).
set utf16_sqlite [sqlite_utf16to16le $utf16le_bom]
set utf16_sqlite [test_translate $utf16le_bom UTF16LE UTF16LE]
do_bincmp_test $testname.5.le.le $utf16_sqlite $utf16le
# Step 5 (big endian to big endian).
set utf16_sqlite [sqlite_utf16to16be $utf16be_bom]
set utf16_sqlite [test_translate $utf16be_bom UTF16 UTF16BE]
do_bincmp_test $testname.5.be.be $utf16_sqlite $utf16be
# Step 5 (big endian to little endian).
set utf16_sqlite [sqlite_utf16to16le $utf16be_bom]
set utf16_sqlite [test_translate $utf16be_bom UTF16 UTF16LE]
do_bincmp_test $testname.5.be.le $utf16_sqlite $utf16le
# Step 5 (little endian to big endian).
set utf16_sqlite [sqlite_utf16to16be $utf16le_bom]
set utf16_sqlite [test_translate $utf16le_bom UTF16 UTF16BE]
do_bincmp_test $testname.5.le.be $utf16_sqlite $utf16be
}
translate_selftest
test_conversion enc-1 "hello world"
test_conversion enc-2 "sqlite"
test_conversion enc-3 ""
test_conversion enc-X "\u0100"
test_conversion enc-4 "\u1234"
test_conversion enc-5 "\u4321abc"
test_conversion enc-6 "\u4321\u1234"
@ -144,4 +146,3 @@ finish_test