Add IN-operator normalizating and the SQLITE_NORMALIZE_CLI compile-time

option for generating a stand-alone program.

FossilOrigin-Name: d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9

ext/misc/normalize.c

@@ -35,11 +35,19 @@
 **
 ** The purpose of normalization is two-fold:
 **
-**   (1)  Sanitize queries by removing possibly sensitive information contained
+**   (1)  Sanitize queries by removing potentially private or sensitive
-**        in literals.
+**        information contained in literals.
 **
 **   (2)  Identify structurally identical queries by comparing their
 **        normalized forms.
+**
+** Command-Line Utility
+** --------------------
+**
+** This file also contains code for a command-line utility that converts
+** SQL queries in text files into their normalized forms.  To build the
+** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
+** and link it against the SQLite library.
 */
 #include <sqlite3.h>
 #include <string.h>
@@ -48,9 +56,13 @@
 ** Implementation note:
 **
 ** Much of the tokenizer logic is copied out of the tokenize.c source file
-** of SQLite.  This logic could be simplified for this particular application,
+** of SQLite.  That logic could be simplified for this particular application,
 ** but that would impose a risk of introducing subtle errors.  It is best to
 ** keep the code as close to the original as possible.
+**
+** The tokenize code is in sync with the SQLite core as of 2018-01-08.
+** Any future changes to the core tokenizer might require corresponding
+** adjustments to the tokenizer logic in this module.
 */
 
 
@@ -572,13 +584,54 @@ char *sqlite3_normalize(const char *zSql){
   while( j>0 && z[j-1]==' ' ){ j--; }
   if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   z[j] = 0;
+
+  /* Make a second pass converting "in(...)" where the "..." is not a
+  ** SELECT statement into "in(?,?,?)" */
+  for(i=0; i<j; i=n){
+    char *zIn = strstr(z+i, "in(");
+    int nParen;
+    if( zIn==0 ) break;
+    n = (int)(zIn-z)+3;  /* Index of first char past "in(" */
+    if( n && IdChar(zIn[-1]) ) continue;
+    if( strncmp(zIn, "in(select",9)==0 && !IdChar(zIn[9]) ) continue;
+    if( strncmp(zIn, "in(with",7)==0 && !IdChar(zIn[7]) ) continue;
+    for(nParen=1, k=0; z[n+k]; k++){
+      if( z[n+k]=='(' ) nParen++;
+      if( z[n+k]==')' ){
+        nParen--;
+        if( nParen==0 ) break;
+      }
+    }
+    /* k is the number of bytes in the "..." within "in(...)" */
+    if( k<5 ){
+      z = sqlite3_realloc64(z, j+(5-k)+1);
+      if( z==0 ) return 0;
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }else if( k>5 ){
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }
+    j = j-k+5;
+    z[j] = 0;
+    memcpy(z+n, "?,?,?", 5);
+  }
   return z;
 }
 
-#ifdef NORMALIZE_TEST
+/*
+** For testing purposes, or to build a stand-alone SQL normalizer program,
+** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
+** it against any SQLite library.  The resulting command-line program will
+** run sqlite3_normalize() over the text of all files named on the command-
+** line and show the result on standard output.
+*/
+#ifdef SQLITE_NORMALIZE_CLI
 #include <stdio.h>
 #include <stdlib.h>
 
+/*
+** Break zIn up into separate SQL statements and run sqlite3_normalize()
+** on each one.  Print the result of each run.
+*/
 static void normalizeFile(char *zIn){
   int i;
   if( zIn==0 ) return;
@@ -604,6 +657,10 @@ static void normalizeFile(char *zIn){
   }
 }
 
+/*
+** The main routine for "sql_normalize".  Read files named on the
+** command-line and run the text of each through sqlite3_normalize().
+*/
 int main(int argc, char **argv){
   int i;
   FILE *in;
@@ -636,4 +693,4 @@ int main(int argc, char **argv){
   }
   sqlite3_free(zBuf);
 }
-#endif /* NORMALIZE_TEST */
+#endif /* SQLITE_NORMALIZE_CLI */

manifest

@@ -1,5 +1,5 @@
-C First\scode\sfor\san\sauxiliary\sfunction\sthat\swill\snormalize\san\sSQL\sstatement.
+C Add\sIN-operator\snormalizating\sand\sthe\sSQLITE_NORMALIZE_CLI\scompile-time\noption\sfor\sgenerating\sa\sstand-alone\sprogram.
-D 2018-01-08T16:54:26.258
+D 2018-01-08T19:18:27.144
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281
 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
-F ext/misc/normalize.c 5ca65b1332e41d679d7c07374398b49d020281d4ea30f267c6a6fb500e71f7c7
+F ext/misc/normalize.c c5f9b3968bf0bfac6ca1c1f802a30ca08e96a9ac7ab3297190b2745046d7d838
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
@@ -1698,10 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
+P 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
-R 9bd921419b4f2c3b5e55f041d45b29ce
+R 52aee785605536a5becd57b22eca4f38
-T *branch * normalize
-T *sym-normalize *
-T -sym-trunk *
 U drh
-Z c8d4afe60d21d90fa2f1cb97ff587c09
+Z be9e0b00c5811d466485564c364b04ac

manifest.uuid

@@ -1 +1 @@
-84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
+d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9