diff --git a/manifest b/manifest index f8f7ebdbb6..68a14a5b76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"index_usage"\sutility\sprogram. -D 2018-12-04T16:51:42.708 +C Enhance\sthe\ssqlite3_normalize_sql()\sinterface\sso\sthat\sit\sworks\seven\sif\sthe\nprepared\sstatement\swas\snot\sinitially\scompiled\susing\nSQLITE_PREPARE_NORMALIZED.\s\sEnhance\sthe\s".trace"\scommand\sin\sthe\sCLI\sso\sthat\nit\sis\sable\sto\saccess\sthe\sfull\sscope\sof\sfunctionality\sprovided\sby\s\nsqlite3_trace_v2()\sand\sin\sparticular\sso\sthat\sit\sis\sable\sto\sshow\snormalized\nSQL\soutput\susing\sthe\snewly\senhanced\ssqlite3_normalize_sql()\sinterface. +D 2018-12-05T13:39:06.092 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -464,8 +464,8 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b -F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f -F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 +F src/hash.c 6d2f67276469384fb8784fb8e962deeaae6832955626468325d705a01b999594 +F src/hash.h eebf2250e56b5d5353b873406557e373d2888cf51f111e28917666456d479e85 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 @@ -502,17 +502,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 +F src/prepare.c 277c8af17124bd4f67ae6e1a8e795a91f12bf7ce3a4b344c7eac46d360250c20 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f -F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c +F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h f54ee1ef1e0f99d27af20561df72aac9158ed420cfc3a2e330fdee40672daf37 +F src/sqliteInt.h 128f2ef4cd6037300cb5510dd3c882417b225d6f4ed6210bddfd920104c7341f F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -581,8 +581,8 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c dc825a6ec99a5066c1aa0d9824509057c0510f03cc8c72f81ba074553f8a5ae8 -F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 +F src/vdbeapi.c 9ac7e3946a2762b79c314922c84c7de30731e0ed1d7c94ac82266b795221d9d7 +F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1280,7 +1280,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d2bf5daeb6f449f0169c9ef3094db17a16a02199c5dcf1a635a3e79b07eb0edd F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494 -F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d +F test/shell4.test a6881d0ae226ded0df8ebdfa574c5aa6dc28d6884ccba1089dc56ed08b9e5ef4 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e -R 14a419221d3a581ec01dad82979f03fc +P df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 +R e65d818662ab9003f3be5e2ff4aec20c U drh -Z 0158a53fd6ddbd828c2808dfe15d0a38 +Z 66f377ac461c094ae8c28a5487556c27 diff --git a/manifest.uuid b/manifest.uuid index 19384dd637..fab3fff857 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 \ No newline at end of file +7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 \ No newline at end of file diff --git a/src/hash.c b/src/hash.c index fba9dc9f80..e725f2aa5e 100644 --- a/src/hash.c +++ b/src/hash.c @@ -72,7 +72,7 @@ static unsigned int strHashN(const char *z, int n){ /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). ** 0x9e3779b1 is 2654435761 which is the closest prime number to ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ - h += sqlite3UpperToLower[z[i]]; + h += sqlite3UpperToLower[(unsigned char)z[i]]; h *= 0x9e3779b1; } return h; diff --git a/src/hash.h b/src/hash.h index 90540eda50..cf773663b7 100644 --- a/src/hash.h +++ b/src/hash.h @@ -68,6 +68,9 @@ struct HashElem { void sqlite3HashInit(Hash*); void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); void *sqlite3HashFind(const Hash*, const char *pKey); +#ifdef SQLITE_ENABLE_NORMALIZE +void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey); +#endif void sqlite3HashClear(Hash*); /* diff --git a/src/prepare.c b/src/prepare.c index 4d33f0b1e1..86afac5350 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -791,8 +791,7 @@ done: */ static int estimateNormalizedSize( const char *zSql, /* The original SQL string */ - int nSql, /* Length of original SQL string */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ + int nSql /* Length of original SQL string */ ){ int nOut = nSql + 4; const char *z = zSql; @@ -847,18 +846,14 @@ static void copyNormalizedToken( } /* -** Perform normalization of the SQL contained in the prepared statement and -** store the result in the zNormSql field. The schema for the associated -** databases are consulted while performing the normalization in order to -** determine if a token appears to be an identifier. All identifiers are -** left intact in the normalized SQL and all literals are replaced with a -** single '?'. +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. */ -void sqlite3Normalize( +char *sqlite3Normalize( Vdbe *pVdbe, /* VM being reprepared */ const char *zSql, /* The original SQL string */ - int nSql, /* Size of the input string in bytes */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ + int nSql /* Size of the input string in bytes */ ){ sqlite3 *db; /* Database handle. */ char *z; /* The output string */ @@ -873,11 +868,10 @@ void sqlite3Normalize( db = sqlite3VdbeDb(pVdbe); assert( db!=0 ); - assert( pVdbe->zNormSql==0 ); - if( zSql==0 ) return; - nZ = estimateNormalizedSize(zSql, nSql, prepFlags); + if( zSql==0 ) return 0; + nZ = estimateNormalizedSize(zSql, nSql); z = sqlite3DbMallocRawNN(db, nZ); - if( z==0 ) return; + if( z==0 ) goto normalizeError; sqlite3HashInit(&inHash); for(i=j=0; i0 && z[j-1]!=';' ){ z[j++] = ';'; } z[j] = 0; assert( jzNormSql = z; sqlite3HashClear(&inHash); + return z; + +normalizeError: + sqlite3DbFree(db, z); + sqlite3HashClear(&inHash); + return 0; } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/shell.c.in b/src/shell.c.in index ad5a1498be..887c2fcf59 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1007,6 +1007,7 @@ struct ShellState { u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ @@ -1066,6 +1067,12 @@ struct ShellState { #define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ #define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + /* ** These are the allowed shellFlgs values */ @@ -3492,7 +3499,22 @@ static const char *(azHelp[]) = { ".testcase NAME Begin redirecting output to 'testcase-out.txt'", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", - ".trace FILE|off Output each SQL statement as it is run", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", @@ -3999,24 +4021,60 @@ static FILE *output_file_open(const char *zFile, int bTextMode){ return f; } -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) +#ifndef SQLITE_OMIT_TRACE /* ** A routine for handling output from sqlite3_trace(). */ static int sql_trace_callback( - unsigned mType, - void *pArg, - void *pP, - void *pX + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ ){ - FILE *f = (FILE*)pArg; - UNUSED_PARAMETER(mType); - UNUSED_PARAMETER(pP); - if( f ){ - const char *z = (const char*)pX; - int i = strlen30(z); - while( i>0 && z[i-1]==';' ){ i--; } - utf8_printf(f, "%.*s;\n", i, z); + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } } return 0; } @@ -7838,23 +7896,55 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else +#ifndef SQLITE_OMIT_TRACE if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; open_db(p, 0); - if( nArg!=2 ){ - raw_printf(stderr, "Usage: .trace FILE|off\n"); - rc = 1; - goto meta_command_exit; + for(jj=1; jjeTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } } - output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) if( p->traceOut==0 ){ sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); } -#endif }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9f9d03a605..fc52fecfe2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4424,7 +4424,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); #ifdef SQLITE_ENABLE_NORMALIZE -void sqlite3Normalize(Vdbe*, const char*, int, u8); +char *sqlite3Normalize(Vdbe*, const char*, int); #endif int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 3b040a5b01..ca073589f7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1712,7 +1712,11 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ */ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return p ? p->zNormSql : 0; + if( p==0 ) return 0; + if( p->zNormSql==0 && p->zSql!=0 ){ + p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql)); + } + return p->zNormSql; } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f1496a3abf..0ed9e81d90 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -67,7 +67,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ #ifdef SQLITE_ENABLE_NORMALIZE assert( p->zNormSql==0 ); if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){ - sqlite3Normalize(p, p->zSql, n, prepFlags); + p->zNormSql = sqlite3Normalize(p, p->zSql, n); assert( p->zNormSql!=0 || p->db->mallocFailed ); } #endif diff --git a/test/shell4.test b/test/shell4.test index 88e5e69a28..2210e74a67 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -107,14 +107,14 @@ SELECT 1; } {1 1 1} do_test shell4-2.1 { - catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace" -} {1 {Usage: .trace FILE|off}} + catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown" +} {1 {Unknown option "--unknown" on ".trace"}} do_test shell4-2.2 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n" } {0 {}} do_test shell4-2.3 { - catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n" -} {/^1 {PRAGMA.*Usage:.*}$/} + catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n" +} {/^0 {PRAGMA.*}$/} ifcapable trace { do_test shell4-2.4 { catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"