Enhance the sqlite3_normalize_sql() interface so that it works even if the

prepared statement was not initially compiled using
SQLITE_PREPARE_NORMALIZED.  Enhance the ".trace" command in the CLI so that
it is able to access the full scope of functionality provided by 
sqlite3_trace_v2() and in particular so that it is able to show normalized
SQL output using the newly enhanced sqlite3_normalize_sql() interface.

FossilOrigin-Name: 7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5
This commit is contained in:
drh 2018-12-05 13:39:06 +00:00
parent 731dd6ebda
commit 707821ff72
10 changed files with 159 additions and 69 deletions

View File

@ -1,5 +1,5 @@
C Add\sthe\s"index_usage"\sutility\sprogram.
D 2018-12-04T16:51:42.708
C Enhance\sthe\ssqlite3_normalize_sql()\sinterface\sso\sthat\sit\sworks\seven\sif\sthe\nprepared\sstatement\swas\snot\sinitially\scompiled\susing\nSQLITE_PREPARE_NORMALIZED.\s\sEnhance\sthe\s".trace"\scommand\sin\sthe\sCLI\sso\sthat\nit\sis\sable\sto\saccess\sthe\sfull\sscope\sof\sfunctionality\sprovided\sby\s\nsqlite3_trace_v2()\sand\sin\sparticular\sso\sthat\sit\sis\sable\sto\sshow\snormalized\nSQL\soutput\susing\sthe\snewly\senhanced\ssqlite3_normalize_sql()\sinterface.
D 2018-12-05T13:39:06.092
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c
@ -464,8 +464,8 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812
F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f
F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b
F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f
F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
F src/hash.c 6d2f67276469384fb8784fb8e962deeaae6832955626468325d705a01b999594
F src/hash.h eebf2250e56b5d5353b873406557e373d2888cf51f111e28917666456d479e85
F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055
@ -502,17 +502,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc
F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354
F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13
F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69
F src/prepare.c 277c8af17124bd4f67ae6e1a8e795a91f12bf7ce3a4b344c7eac46d360250c20
F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381
F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1
F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f
F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c
F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d
F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
F src/sqliteInt.h f54ee1ef1e0f99d27af20561df72aac9158ed420cfc3a2e330fdee40672daf37
F src/sqliteInt.h 128f2ef4cd6037300cb5510dd3c882417b225d6f4ed6210bddfd920104c7341f
F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@ -581,8 +581,8 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d
F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b
F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907
F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980
F src/vdbeapi.c dc825a6ec99a5066c1aa0d9824509057c0510f03cc8c72f81ba074553f8a5ae8
F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601
F src/vdbeapi.c 9ac7e3946a2762b79c314922c84c7de30731e0ed1d7c94ac82266b795221d9d7
F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4
F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9
F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f
@ -1280,7 +1280,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
F test/shell1.test d2bf5daeb6f449f0169c9ef3094db17a16a02199c5dcf1a635a3e79b07eb0edd
F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494
F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
F test/shell4.test a6881d0ae226ded0df8ebdfa574c5aa6dc28d6884ccba1089dc56ed08b9e5ef4
F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e
R 14a419221d3a581ec01dad82979f03fc
P df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94
R e65d818662ab9003f3be5e2ff4aec20c
U drh
Z 0158a53fd6ddbd828c2808dfe15d0a38
Z 66f377ac461c094ae8c28a5487556c27

View File

@ -1 +1 @@
df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94
7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5

View File

@ -72,7 +72,7 @@ static unsigned int strHashN(const char *z, int n){
/* Knuth multiplicative hashing. (Sorting & Searching, p. 510).
** 0x9e3779b1 is 2654435761 which is the closest prime number to
** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
h += sqlite3UpperToLower[z[i]];
h += sqlite3UpperToLower[(unsigned char)z[i]];
h *= 0x9e3779b1;
}
return h;

View File

@ -68,6 +68,9 @@ struct HashElem {
void sqlite3HashInit(Hash*);
void *sqlite3HashInsert(Hash*, const char *pKey, void *pData);
void *sqlite3HashFind(const Hash*, const char *pKey);
#ifdef SQLITE_ENABLE_NORMALIZE
void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey);
#endif
void sqlite3HashClear(Hash*);
/*

View File

@ -791,8 +791,7 @@ done:
*/
static int estimateNormalizedSize(
const char *zSql, /* The original SQL string */
int nSql, /* Length of original SQL string */
u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */
int nSql /* Length of original SQL string */
){
int nOut = nSql + 4;
const char *z = zSql;
@ -847,18 +846,14 @@ static void copyNormalizedToken(
}
/*
** Perform normalization of the SQL contained in the prepared statement and
** store the result in the zNormSql field. The schema for the associated
** databases are consulted while performing the normalization in order to
** determine if a token appears to be an identifier. All identifiers are
** left intact in the normalized SQL and all literals are replaced with a
** single '?'.
** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return
** the normalization in space obtained from sqlite3DbMalloc(). Or return
** NULL if anything goes wrong or if zSql is NULL.
*/
void sqlite3Normalize(
char *sqlite3Normalize(
Vdbe *pVdbe, /* VM being reprepared */
const char *zSql, /* The original SQL string */
int nSql, /* Size of the input string in bytes */
u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */
int nSql /* Size of the input string in bytes */
){
sqlite3 *db; /* Database handle. */
char *z; /* The output string */
@ -873,11 +868,10 @@ void sqlite3Normalize(
db = sqlite3VdbeDb(pVdbe);
assert( db!=0 );
assert( pVdbe->zNormSql==0 );
if( zSql==0 ) return;
nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
if( zSql==0 ) return 0;
nZ = estimateNormalizedSize(zSql, nSql);
z = sqlite3DbMallocRawNN(db, nZ);
if( z==0 ) return;
if( z==0 ) goto normalizeError;
sqlite3HashInit(&inHash);
for(i=j=0; i<nSql && zSql[i]; i+=n){
int flags = 0;
@ -888,9 +882,7 @@ void sqlite3Normalize(
break;
}
case TK_ILLEGAL: {
sqlite3DbFree(db, z);
sqlite3HashClear(&inHash);
return;
goto normalizeError;
}
case TK_STRING:
case TK_INTEGER:
@ -971,11 +963,7 @@ void sqlite3Normalize(
}
if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
if( rc!=SQLITE_OK ){
sqlite3DbFree(db, z);
sqlite3HashClear(&inHash);
return;
}
if( rc!=SQLITE_OK ) goto normalizeError;
if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
z[j++] = '?';
break;
@ -992,8 +980,13 @@ void sqlite3Normalize(
if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
z[j] = 0;
assert( j<nZ && "two" );
pVdbe->zNormSql = z;
sqlite3HashClear(&inHash);
return z;
normalizeError:
sqlite3DbFree(db, z);
sqlite3HashClear(&inHash);
return 0;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

View File

@ -1007,6 +1007,7 @@ struct ShellState {
u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */
u8 nEqpLevel; /* Depth of the EQP output graph */
u8 eTraceType; /* SHELL_TRACE_* value for type of trace */
unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */
int outCount; /* Revert to stdout when reaching zero */
int cnt; /* Number of records displayed so far */
@ -1066,6 +1067,12 @@ struct ShellState {
#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */
#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */
/* Allowed values for ShellState.eTraceType
*/
#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */
#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */
#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */
/*
** These are the allowed shellFlgs values
*/
@ -3492,7 +3499,22 @@ static const char *(azHelp[]) = {
".testcase NAME Begin redirecting output to 'testcase-out.txt'",
".timeout MS Try opening locked tables for MS milliseconds",
".timer on|off Turn SQL timer on or off",
".trace FILE|off Output each SQL statement as it is run",
#ifndef SQLITE_OMIT_TRACE
".trace ?OPTIONS? Output each SQL statement as it is run",
" FILE Send output to FILE",
" stdout Send output to stdout",
" stderr Send output to stderr",
" off Disable tracing",
" --expanded Expand query parameters",
#ifdef SQLITE_ENABLE_NORMALIZE
" --normalized Normal the SQL statements",
#endif
" --plain Show SQL as it is input",
" --stmt Trace statement execution (SQLITE_TRACE_STMT)",
" --profile Profile statements (SQLITE_TRACE_PROFILE)",
" --row Trace each row (SQLITE_TRACE_ROW)",
" --close Trace connection close (SQLITE_TRACE_CLOSE)",
#endif /* SQLITE_OMIT_TRACE */
".vfsinfo ?AUX? Information about the top-level VFS",
".vfslist List all available VFSes",
".vfsname ?AUX? Print the name of the VFS stack",
@ -3999,24 +4021,60 @@ static FILE *output_file_open(const char *zFile, int bTextMode){
return f;
}
#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
#ifndef SQLITE_OMIT_TRACE
/*
** A routine for handling output from sqlite3_trace().
*/
static int sql_trace_callback(
unsigned mType,
void *pArg,
void *pP,
void *pX
unsigned mType, /* The trace type */
void *pArg, /* The ShellState pointer */
void *pP, /* Usually a pointer to sqlite_stmt */
void *pX /* Auxiliary output */
){
FILE *f = (FILE*)pArg;
UNUSED_PARAMETER(mType);
UNUSED_PARAMETER(pP);
if( f ){
const char *z = (const char*)pX;
int i = strlen30(z);
while( i>0 && z[i-1]==';' ){ i--; }
utf8_printf(f, "%.*s;\n", i, z);
ShellState *p = (ShellState*)pArg;
sqlite3_stmt *pStmt;
const char *zSql;
int nSql;
if( p->traceOut==0 ) return 0;
if( mType==SQLITE_TRACE_CLOSE ){
utf8_printf(p->traceOut, "-- closing database connection\n");
return 0;
}
if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
zSql = (const char*)pX;
}else{
pStmt = (sqlite3_stmt*)pP;
switch( p->eTraceType ){
case SHELL_TRACE_EXPANDED: {
zSql = sqlite3_expanded_sql(pStmt);
break;
}
#ifdef SQLITE_ENABLE_NORMALIZE
case SHELL_TRACE_NORMALIZED: {
zSql = sqlite3_normalized_sql(pStmt);
break;
}
#endif
default: {
zSql = sqlite3_sql(pStmt);
break;
}
}
}
if( zSql==0 ) return 0;
nSql = strlen30(zSql);
while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
switch( mType ){
case SQLITE_TRACE_ROW:
case SQLITE_TRACE_STMT: {
utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
break;
}
case SQLITE_TRACE_PROFILE: {
sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
break;
}
}
return 0;
}
@ -7838,23 +7896,55 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
#ifndef SQLITE_OMIT_TRACE
if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
int mType = 0;
int jj;
open_db(p, 0);
if( nArg!=2 ){
raw_printf(stderr, "Usage: .trace FILE|off\n");
rc = 1;
goto meta_command_exit;
for(jj=1; jj<nArg; jj++){
const char *z = azArg[jj];
if( z[0]=='-' ){
if( optionMatch(z, "expanded") ){
p->eTraceType = SHELL_TRACE_EXPANDED;
}
#ifdef SQLITE_ENABLE_NORMALIZE
else if( optionMatch(z, "normalized") ){
p->eTraceType = SHELL_TRACE_NORMALIZED;
}
#endif
else if( optionMatch(z, "plain") ){
p->eTraceType = SHELL_TRACE_PLAIN;
}
else if( optionMatch(z, "profile") ){
mType |= SQLITE_TRACE_PROFILE;
}
else if( optionMatch(z, "row") ){
mType |= SQLITE_TRACE_ROW;
}
else if( optionMatch(z, "stmt") ){
mType |= SQLITE_TRACE_STMT;
}
else if( optionMatch(z, "close") ){
mType |= SQLITE_TRACE_CLOSE;
}
else {
raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
rc = 1;
goto meta_command_exit;
}
}else{
output_file_close(p->traceOut);
p->traceOut = output_file_open(azArg[1], 0);
}
}
output_file_close(p->traceOut);
p->traceOut = output_file_open(azArg[1], 0);
#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
if( p->traceOut==0 ){
sqlite3_trace_v2(p->db, 0, 0, 0);
}else{
sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
if( mType==0 ) mType = SQLITE_TRACE_STMT;
sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
}
#endif
}else
#endif /* !defined(SQLITE_OMIT_TRACE) */
#if SQLITE_USER_AUTHENTICATION
if( c=='u' && strncmp(azArg[0], "user", n)==0 ){

View File

@ -4424,7 +4424,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
void sqlite3ParserReset(Parse*);
#ifdef SQLITE_ENABLE_NORMALIZE
void sqlite3Normalize(Vdbe*, const char*, int, u8);
char *sqlite3Normalize(Vdbe*, const char*, int);
#endif
int sqlite3Reprepare(Vdbe*);
void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);

View File

@ -1712,7 +1712,11 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){
*/
const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
Vdbe *p = (Vdbe *)pStmt;
return p ? p->zNormSql : 0;
if( p==0 ) return 0;
if( p->zNormSql==0 && p->zSql!=0 ){
p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
}
return p->zNormSql;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

View File

@ -67,7 +67,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){
#ifdef SQLITE_ENABLE_NORMALIZE
assert( p->zNormSql==0 );
if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
sqlite3Normalize(p, p->zSql, n, prepFlags);
p->zNormSql = sqlite3Normalize(p, p->zSql, n);
assert( p->zNormSql!=0 || p->db->mallocFailed );
}
#endif

View File

@ -107,14 +107,14 @@ SELECT 1;
} {1 1 1}
do_test shell4-2.1 {
catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
} {1 {Usage: .trace FILE|off}}
catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
} {1 {Unknown option "--unknown" on ".trace"}}
do_test shell4-2.2 {
catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
} {0 {}}
do_test shell4-2.3 {
catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
} {/^1 {PRAGMA.*Usage:.*}$/}
catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
} {/^0 {PRAGMA.*}$/}
ifcapable trace {
do_test shell4-2.4 {
catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"