From 9c987a833cf767c7d07c071c25b353623e60ad2a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 21 Jun 2016 10:34:41 +0000
Subject: [PATCH] Update the sqldiff utility so that if the --vtab switch is
 specified "rtree", "fts3", "fts4" and "fts5" tables are diff'd directly and
 the underlying real database tables ignored. Without this switch, all virtual
 tables are ignored and the diff is performed on the underlying real tables.

FossilOrigin-Name: 5d0a9d4c45730e47576bc77373fa7075a74051a5
---
 ext/rbu/rbudiff.test |  69 +++++++++++++++++++-
 manifest             |  18 ++---
 manifest.uuid        |   2 +-
 tool/sqldiff.c       | 152 ++++++++++++++++++++++++++++++++++++++++---
 4 files changed, 221 insertions(+), 20 deletions(-)

diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test
index 0863f6e807..fa8de319dd 100644
--- a/ext/rbu/rbudiff.test
+++ b/ext/rbu/rbudiff.test
@@ -25,6 +25,10 @@ proc get_rbudiff_sql {db1 db2} {
   exec $::PROG --rbu $db1 $db2
 }
 
+proc get_vtab_rbudiff_sql {db1 db2} {
+  exec $::PROG --vtab --rbu $db1 $db2
+}
+
 proc step_rbu {target rbu} {
   while 1 {
     sqlite3rbu rbu $target $rbu
@@ -44,6 +48,11 @@ proc apply_rbudiff {sql target} {
   step_rbu $target rbu.db
 }
 
+proc sqlesc {id} {
+  set ret "'[string map {' ''} $id]'"
+  set ret
+}
+
 # The only argument is the output of an [sqldiff -rbu] run. This command
 # tests that the contents of the rbu_count table is correct. An exception
 # is thrown if it is not.
@@ -54,7 +63,7 @@ proc test_rbucount {sql} {
   tmpdb eval {
     SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table'
   } {
-    set a [tmpdb eval "SELECT count(*) FROM $name"]
+    set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"]
     set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}]
     if {$a != $b} { 
       tmpdb close
@@ -71,9 +80,11 @@ proc rbudiff_cksum {db1} {
   sqlite3 dbtmp $db1
   foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
     set cols [list]
-    dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" }
+    dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { 
+      lappend cols "quote( $name )" 
+    }
     append txt [dbtmp eval \
-      "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1"
+      "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1"
     ]
   }
   dbtmp close
@@ -159,5 +170,57 @@ foreach {tn init mod} {
   do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
 }
 
+#-------------------------------------------------------------------------
+# Test that if the --vtab switch is present, [sqldiff] handles virtual
+# table types fts[345] and rtree correctly.
+#
+ifcapable fts3&&fts5&&rtree {
+
+foreach {tn init mod} {
+  1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(c);
+    INSERT INTO t1 VALUES('a b c');
+    INSERT INTO t1 VALUES('a b c');
+  } {
+    DELETE FROM t1 WHERE rowid = 1;
+    INSERT INTO t1 VALUES('a b c');
+  }
+
+  2 {
+    CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2);
+    INSERT INTO "x y" VALUES(1, 2, 3);
+    INSERT INTO "x y" VALUES(2, 4, 6);
+  } {
+    DELETE FROM "x y" WHERE rowid = 1;
+    INSERT INTO "x y" VALUES(3, 6, 9);
+  }
+
+  3 {
+    CREATE VIRTUAL TABLE 'x''y' USING fts3;
+    INSERT INTO 'x''y' VALUES('one two three');
+    INSERT INTO 'x''y' VALUES('four five six');
+  } {
+    DELETE FROM 'x''y' WHERE rowid = 1;
+    INSERT INTO 'x''y' VALUES('one two three');
+  }
+
+} {
+
+  forcedelete test.db test.db2
+  sqlite3 db test.db
+  db eval "$init"
+  sqlite3 db test.db2
+  db eval "$init ; $mod"
+  db close
+
+  do_test 2.$tn.1 {
+    set sql [get_vtab_rbudiff_sql test.db test.db2]
+    apply_rbudiff $sql test.db
+  } {SQLITE_DONE}
+  do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]
+}
+
+}
+
 finish_test
 
diff --git a/manifest b/manifest
index f7bf4fc9ff..12c06cd44f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C For\sa\stable\son\sthe\srhs\sof\sa\sLEFT\sJOIN\soperator,\sdo\snot\sinclude\sterms\slike\s"IS\sNULL"\sfrom\sthe\sWHERE\sclause\sin\sthe\scursor-hint.\sThese\smay\sbe\sfalse\sfor\srows\sthat\sthe\scursor\swould\sotherwise\svisit,\sbut\strue\sfor\sa\srow\sof\sall\sNULL\svalues\sgenerated\sby\sthe\sLEFT\sJOIN.
-D 2016-06-20T17:25:50.750
+C Update\sthe\ssqldiff\sutility\sso\sthat\sif\sthe\s--vtab\sswitch\sis\sspecified\s"rtree",\s"fts3",\s"fts4"\sand\s"fts5"\stables\sare\sdiff'd\sdirectly\sand\sthe\sunderlying\sreal\sdatabase\stables\signored.\sWithout\sthis\sswitch,\sall\svirtual\stables\sare\signored\sand\sthe\sdiff\sis\sperformed\son\sthe\sunderlying\sreal\stables.
+D 2016-06-21T10:34:41.782
 F Makefile.in f3f7d2060ce03af4584e711ef3a626ef0b1d6340
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 50149765ef72f4e652b9a0f1f6462c4784bb9423
@@ -243,7 +243,7 @@ F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2
 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831
 F ext/rbu/rbu_common.tcl 3a4b916b6f5dca9c9da9a30863e272fe5ea4414f
 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695
-F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091
+F ext/rbu/rbudiff.test 4c9f8df6f723f553781d3d117501b7e9d170a145
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
@@ -1469,7 +1469,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c ca315aca4e2d24233e8f2000edea5880c53d1875
+F tool/sqldiff.c 131e9c6e09d3d78e3c4ae8b5fd23451521723aac
 F tool/srcck1.c 4c39bdfa9a92edd20233ee720df84dbeb2417602
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1502,8 +1502,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0b1579caf06a2c42433b8bc9dc28c9ad381aa07c 7455d932f5079ffe40462a8c119fc22b8a9bcbcc
-R 6b36cf1cbd5dc0e94253a2a90e40c230
-T +closed 7455d932f5079ffe40462a8c119fc22b8a9bcbcc
+P 913e595615e2ef40fb431f6e7678f6fc8439782e
+R a344dc9988f1497ebec3b8ad168c1397
+T *branch * sqldiff-vtab-support
+T *sym-sqldiff-vtab-support *
+T -sym-trunk *
 U dan
-Z 537cf80c06fd3994884c97d3059d4966
+Z 6909c0a252ed2326d78857113119f0ab
diff --git a/manifest.uuid b/manifest.uuid
index 53806979d6..677a251025 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-913e595615e2ef40fb431f6e7678f6fc8439782e
\ No newline at end of file
+5d0a9d4c45730e47576bc77373fa7075a74051a5
\ No newline at end of file
diff --git a/tool/sqldiff.c b/tool/sqldiff.c
index 319acafdbd..4a5389c603 100644
--- a/tool/sqldiff.c
+++ b/tool/sqldiff.c
@@ -33,6 +33,7 @@ struct GlobalVars {
   const char *zArgv0;       /* Name of program */
   int bSchemaOnly;          /* Only show schema differences */
   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
+  int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
   unsigned fDebug;          /* Debug flags */
   sqlite3 *db;              /* The database connection */
 } g;
@@ -1735,6 +1736,144 @@ end_changeset_one_table:
   sqlite3_free(zId);
 }
 
+/*
+** Extract the next SQL keyword or quoted string from buffer zIn and copy it
+** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
+** Return a pointer to the character within zIn immediately following 
+** the token or quoted string just extracted.
+*/
+const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
+  const char *p = zIn;
+  char *pOut = zBuf;
+  char *pEnd = &pOut[nBuf-1];
+  char q = 0;                     /* quote character, if any */
+
+  if( p==0 ) return 0;
+  while( *p==' ' ) p++;
+  switch( *p ){
+    case '"': q = '"'; break;
+    case '\'': q = '\''; break;
+    case '`': q = '`'; break;
+    case '[': q = ']'; break;
+  }
+
+  if( q ){
+    p++;
+    while( *p && pOut<pEnd ){
+      if( *p==q ){
+        p++;
+        if( *p!=q ) break;
+      }
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }else{
+    while( *p && *p!=' ' && *p!='(' ){
+      if( pOut<pEnd ) *pOut++ = *p;
+      p++;
+    }
+  }
+
+  *pOut = '\0';
+  return p;
+}
+
+/*
+** This function is the implementation of SQL scalar function "module_name":
+**
+**   module_name(SQL)
+**
+** The only argument should be an SQL statement of the type that may appear
+** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
+** statement, then the value returned is the name of the module that it
+** uses. Otherwise, if the statement is not a CVT, NULL is returned.
+*/
+static void module_name_func(
+  sqlite3_context *pCtx, 
+  int nVal, sqlite3_value **apVal
+){
+  const char *zSql;
+  char zToken[32];
+
+  assert( nVal==1 );
+  zSql = (const char*)sqlite3_value_text(apVal[0]);
+
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
+  zSql = gobble_token(zSql, zToken, sizeof(zToken));
+  
+  sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
+}
+
+/*
+** Return the text of an SQL statement that itself returns the list of
+** tables to process within the database.
+*/
+const char *all_tables_sql(){
+  if( g.bHandleVtab ){
+    int rc;
+  
+    rc = sqlite3_exec(g.db, 
+        "CREATE TEMP TABLE tblmap(module, postfix);"
+        "INSERT INTO temp.tblmap VALUES"
+        "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
+  
+        "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
+        "('fts4', '_docsize'), ('fts4', '_stat'),"
+  
+        "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
+        "('fts5', '_docsize'), ('fts5', '_config'),"
+  
+        "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
+        , 0, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    rc = sqlite3_create_function(
+        g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
+    );
+    assert( rc==SQLITE_OK );
+  
+    return 
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM main.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      "UNION \n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND (\n"
+      "    module_name(sql) IS NULL OR \n"
+      "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
+      " ) AND name NOT IN (\n"
+      "  SELECT a.name || b.postfix \n"
+        "FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
+        "WHERE module_name(a.sql) = b.module\n" 
+      " )\n"
+      " ORDER BY name";
+  }else{
+    return
+      "SELECT name FROM main.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " UNION\n"
+      "SELECT name FROM aux.sqlite_master\n"
+      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
+      " ORDER BY name";
+  }
+}
+
 /*
 ** Print sketchy documentation for this utility program
 */
@@ -1751,6 +1890,7 @@ static void showHelp(void){
 "  --summary             Show only a summary of the differences\n"
 "  --table TAB           Show only differences in table TAB\n"
 "  --transaction         Show SQL output inside a transaction\n"
+"  --vtab                Handle fts3, fts4, fts5 and rtree tables\n"
   );
 }
 
@@ -1821,6 +1961,9 @@ int main(int argc, char **argv){
       if( strcmp(z,"transaction")==0 ){
         useTransaction = 1;
       }else
+      if( strcmp(z,"vtab")==0 ){
+        g.bHandleVtab = 1;
+      }else
       {
         cmdlineError("unknown option: %s", argv[i]);
       }
@@ -1875,14 +2018,7 @@ int main(int argc, char **argv){
     xDiff(zTab, out);
   }else{
     /* Handle tables one by one */
-    pStmt = db_prepare(
-      "SELECT name FROM main.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " UNION\n"
-      "SELECT name FROM aux.sqlite_master\n"
-      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
-      " ORDER BY name"
-    );
+    pStmt = db_prepare( all_tables_sql() );
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       xDiff((const char*)sqlite3_column_text(pStmt,0), out);
     }