diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c index 914a154d9a..fe5fb01b4d 100644 --- a/ext/misc/normalize.c +++ b/ext/misc/normalize.c @@ -35,11 +35,19 @@ ** ** The purpose of normalization is two-fold: ** -** (1) Sanitize queries by removing possibly sensitive information contained -** in literals. +** (1) Sanitize queries by removing potentially private or sensitive +** information contained in literals. ** ** (2) Identify structurally identical queries by comparing their ** normalized forms. +** +** Command-Line Utility +** -------------------- +** +** This file also contains code for a command-line utility that converts +** SQL queries in text files into their normalized forms. To build the +** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI +** and link it against the SQLite library. */ #include #include @@ -48,9 +56,13 @@ ** Implementation note: ** ** Much of the tokenizer logic is copied out of the tokenize.c source file -** of SQLite. This logic could be simplified for this particular application, +** of SQLite. That logic could be simplified for this particular application, ** but that would impose a risk of introducing subtle errors. It is best to ** keep the code as close to the original as possible. +** +** The tokenize code is in sync with the SQLite core as of 2018-01-08. +** Any future changes to the core tokenizer might require corresponding +** adjustments to the tokenizer logic in this module. */ @@ -572,13 +584,54 @@ char *sqlite3_normalize(const char *zSql){ while( j>0 && z[j-1]==' ' ){ j--; } if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; } z[j] = 0; + + /* Make a second pass converting "in(...)" where the "..." is not a + ** SELECT statement into "in(?,?,?)" */ + for(i=0; i5 ){ + memmove(z+n+5, z+n+k, j-(n+k)); + } + j = j-k+5; + z[j] = 0; + memcpy(z+n, "?,?,?", 5); + } return z; } -#ifdef NORMALIZE_TEST +/* +** For testing purposes, or to build a stand-alone SQL normalizer program, +** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link +** it against any SQLite library. The resulting command-line program will +** run sqlite3_normalize() over the text of all files named on the command- +** line and show the result on standard output. +*/ +#ifdef SQLITE_NORMALIZE_CLI #include #include +/* +** Break zIn up into separate SQL statements and run sqlite3_normalize() +** on each one. Print the result of each run. +*/ static void normalizeFile(char *zIn){ int i; if( zIn==0 ) return; @@ -604,6 +657,10 @@ static void normalizeFile(char *zIn){ } } +/* +** The main routine for "sql_normalize". Read files named on the +** command-line and run the text of each through sqlite3_normalize(). +*/ int main(int argc, char **argv){ int i; FILE *in; @@ -636,4 +693,4 @@ int main(int argc, char **argv){ } sqlite3_free(zBuf); } -#endif /* NORMALIZE_TEST */ +#endif /* SQLITE_NORMALIZE_CLI */ diff --git a/manifest b/manifest index 7cd42ecfd6..4cc5e208a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\scode\sfor\san\sauxiliary\sfunction\sthat\swill\snormalize\san\sSQL\sstatement. -D 2018-01-08T16:54:26.258 +C Add\sIN-operator\snormalizating\sand\sthe\sSQLITE_NORMALIZE_CLI\scompile-time\noption\sfor\sgenerating\sa\sstand-alone\sprogram. +D 2018-01-08T19:18:27.144 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb @@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 -F ext/misc/normalize.c 5ca65b1332e41d679d7c07374398b49d020281d4ea30f267c6a6fb500e71f7c7 +F ext/misc/normalize.c c5f9b3968bf0bfac6ca1c1f802a30ca08e96a9ac7ab3297190b2745046d7d838 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c @@ -1698,10 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927 -R 9bd921419b4f2c3b5e55f041d45b29ce -T *branch * normalize -T *sym-normalize * -T -sym-trunk * +P 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea +R 52aee785605536a5becd57b22eca4f38 U drh -Z c8d4afe60d21d90fa2f1cb97ff587c09 +Z be9e0b00c5811d466485564c364b04ac diff --git a/manifest.uuid b/manifest.uuid index 68c796f48f..6aaff97c39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea \ No newline at end of file +d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9 \ No newline at end of file