From e611f1447132d3d34c7cf83e66a2a5a3abdf518e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Mar 2017 11:44:00 +0000 Subject: [PATCH] Add the --preserve-rowids option to the ".dump" command in the CLI. FossilOrigin-Name: c60aee24714a47ce12ee2a4dcefb9f55211d3761 --- manifest | 16 +-- manifest.uuid | 2 +- src/shell.c | 352 ++++++++++++++++++++++++++++++++--------------- test/shell1.test | 48 ++++++- 4 files changed, 295 insertions(+), 123 deletions(-) diff --git a/manifest b/manifest index a3944b0665..4a60c5aefa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sscript\sext/rbu/rbu_round_trip.tcl.\sUses\s"dbselftest"\sto\stest\sthat\n"rbu"\sand\s"sqldiff"\swork\stogether. -D 2017-03-07T20:03:25.030 +C Add\sthe\s--preserve-rowids\soption\sto\sthe\s".dump"\scommand\sin\sthe\sCLI. +D 2017-03-08T11:44:00.069 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -399,7 +399,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c d12f3539f80db38b09015561b569e0eb1c4b6c5f -F src/shell.c 27d2b31099fd2cd749e44d025ef9b54ca26692cb +F src/shell.c c42c3031f715712a0cd47d8f08bd2d1dfec8baa0 F src/sqlite.h.in 4d0c08f8640c586564a7032b259c5f69bf397850 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae @@ -1110,7 +1110,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 52ac23a345772ab0d6d3241a21a633fdaa3ed581 +F test/shell1.test 5df739ee2fe5c9dbbb2f9a0158b9723bda0910b8 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d @@ -1563,7 +1563,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2cb71583d631cd417acbeebbb4ee950573a9deef -R ee3baff0fd57ede16b35b4777ef32cf9 -U dan -Z 9c10ed94bada1b1224e7c4aff135f1d6 +P 961e79da73b4550b3e5b0f9a617133a76485db67 +R c754df2de2ef21ae6643ebeaf45e8d82 +U drh +Z bce538a1e35be638681ea14d388ad037 diff --git a/manifest.uuid b/manifest.uuid index 744c25a1e5..7002a8bc75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -961e79da73b4550b3e5b0f9a617133a76485db67 \ No newline at end of file +c60aee24714a47ce12ee2a4dcefb9f55211d3761 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2dcf915f28..07f977d46a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -455,28 +455,6 @@ static int isNumber(const char *z, int *realnum){ return *z==0; } -/* -** A global char* and an SQL function to access its current value -** from within an SQL statement. This program used to use the -** sqlite_exec_printf() API to substitue a string into an SQL statement. -** The correct way to do this with sqlite3 is to use the bind API, but -** since the shell is built around the callback paradigm it would be a lot -** of work. Instead just use this hack, which is quite harmless. -*/ -static const char *zShellStatic = 0; -static void shellstaticFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - assert( 0==argc ); - assert( zShellStatic ); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - sqlite3_result_text(context, zShellStatic, -1, SQLITE_STATIC); -} - - /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -618,6 +596,7 @@ struct ShellState { int countChanges; /* True to display change counts */ int backslashOn; /* Resolve C-style \x escapes in SQL input text */ int outCount; /* Revert to stdout when reaching zero */ + int preserveRowid; /* Preserver ROWID values on a ".dump" command */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ @@ -1333,6 +1312,16 @@ static void set_table_name(ShellState *p, const char *zName){ z[n] = 0; } +/* +** A variable length string to which one can append text. +*/ +typedef struct ShellString ShellString; +struct ShellString { + char *z; + int n; + int nAlloc; +}; + /* zIn is either a pointer to a NULL-terminated string in memory obtained ** from malloc(), or a NULL pointer. The string pointed to by zAppend is ** added to zIn, and the result returned in memory obtained from malloc(). @@ -1341,13 +1330,12 @@ static void set_table_name(ShellState *p, const char *zName){ ** If the third argument, quote, is not '\0', then it is used as a ** quote character for zAppend. */ -static char *appendText(char *zIn, char const *zAppend, char quote){ +static void appendText(ShellString *p, char const *zAppend, char quote){ int len; int i; int nAppend = strlen30(zAppend); - int nIn = (zIn?strlen30(zIn):0); - len = nAppend+nIn+1; + len = nAppend+p->n+1; if( quote ){ len += 2; for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ){ + memset(p, 0, sizeof(*p)); + return; + } } if( quote ){ - char *zCsr = &zIn[nIn]; + char *zCsr = p->z+p->n; *zCsr++ = quote; for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; }else{ - memcpy(&zIn[nIn], zAppend, nAppend); - zIn[len-1] = '\0'; + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; } - - return zIn; } @@ -2024,6 +2015,124 @@ static int shell_exec( return rc; } +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(ShellState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = p->preserveRowid; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + nAlloc = nAlloc*2 + nCol + 10; + azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azCol==0 ){ + raw_printf(stderr, "Error: out of memory\n"); + exit(1); + } + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + azCol[0] = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( preserveRowid && isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + int rc; + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; +} + + + /* ** This is a different callback routine used for dumping the database. @@ -2036,7 +2145,6 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ const char *zTable; const char *zType; const char *zSql; - const char *zPrepStmt = 0; ShellState *p = (ShellState *)pArg; UNUSED_PARAMETER(azCol); @@ -2046,7 +2154,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ zSql = azArg[2]; if( strcmp(zTable, "sqlite_sequence")==0 ){ - zPrepStmt = "DELETE FROM sqlite_sequence;\n"; + raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ raw_printf(p->out, "ANALYZE sqlite_master;\n"); }else if( strncmp(zTable, "sqlite_", 7)==0 ){ @@ -2069,58 +2177,64 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } if( strcmp(zType, "table")==0 ){ - sqlite3_stmt *pTableInfo = 0; - char *zSelect = 0; - char *zTableInfo = 0; - char *zTmp = 0; - int nRow = 0; + ShellString sSelect; + ShellString sTable; + char **azCol; + int i; + char *savedDestTable; + int savedMode; - zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); - zTableInfo = appendText(zTableInfo, zTable, '"'); - zTableInfo = appendText(zTableInfo, ");", 0); - - rc = sqlite3_prepare_v2(p->db, zTableInfo, -1, &pTableInfo, 0); - free(zTableInfo); - if( rc!=SQLITE_OK || !pTableInfo ){ - return 1; + azCol = tableColumnList(p, zTable); + if( azCol==0 ){ + p->nErr++; + return 0; } - zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); /* Always quote the table name, even if it appears to be pure ascii, ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - zTmp = appendText(zTmp, zTable, '"'); - if( zTmp ){ - zSelect = appendText(zSelect, zTmp, '\''); - free(zTmp); - } - zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); - rc = sqlite3_step(pTableInfo); - while( rc==SQLITE_ROW ){ - const char *zText = (const char *)sqlite3_column_text(pTableInfo, 1); - zSelect = appendText(zSelect, "quote(", 0); - zSelect = appendText(zSelect, zText, '"'); - rc = sqlite3_step(pTableInfo); - if( rc==SQLITE_ROW ){ - zSelect = appendText(zSelect, "), ", 0); - }else{ - zSelect = appendText(zSelect, ") ", 0); + memset(&sTable, 0, sizeof(sTable)); + appendText(&sTable, zTable, '"'); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], '"'); } - nRow++; + appendText(&sTable, ")", 0); } - rc = sqlite3_finalize(pTableInfo); - if( rc!=SQLITE_OK || nRow==0 ){ - free(zSelect); - return 1; - } - zSelect = appendText(zSelect, "|| ')' FROM ", 0); - zSelect = appendText(zSelect, zTable, '"'); - rc = run_table_dump_query(p, zSelect, zPrepStmt); - if( rc==SQLITE_CORRUPT ){ - zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); - run_table_dump_query(p, zSelect, 0); + /* Build an appropriate SELECT statement */ + memset(&sSelect, 0, sizeof(sSelect)); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); } - free(zSelect); + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], 0); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, '"'); + + savedDestTable = p->zDestTable; + savedMode = p->mode; + p->zDestTable = sTable.z; + p->mode = p->cMode = MODE_Insert; + rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); + p->zDestTable = savedDestTable; + p->mode = savedMode; + free(sTable.z); + free(sSelect.z); + if( rc ) p->nErr++; } return 0; } @@ -2425,10 +2539,6 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_initialize(); sqlite3_open(p->zDbFilename, &p->db); globalDb = p->db; - if( p->db && sqlite3_errcode(p->db)==SQLITE_OK ){ - sqlite3_create_function(p->db, "shellstatic", 0, SQLITE_UTF8, 0, - shellstaticFunc, 0, 0); - } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", p->zDbFilename, sqlite3_errmsg(p->db)); @@ -3722,21 +3832,42 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + const char *zLike = 0; + int i; + p->preserveRowid = 0; + for(i=1; ipreserveRowid = 1; + }else + { + raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]); + rc = 1; + goto meta_command_exit; + } + }else if( zLike ){ + raw_printf(stderr, "Usage: .dump ?--preserve-rowids? ?LIKE-PATTERN?\n"); + rc = 1; + goto meta_command_exit; + }else{ + zLike = azArg[i]; + } + } open_db(p, 0); /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - if( nArg!=1 && nArg!=2 ){ - raw_printf(stderr, "Usage: .dump ?LIKE-PATTERN?\n"); - rc = 1; - goto meta_command_exit; - } raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n"); raw_printf(p->out, "BEGIN TRANSACTION;\n"); p->writableSchema = 0; + /* Set writable_schema=ON since doing so forces SQLite to initialize + ** as much of the schema as it can even if the sqlite_master table is + ** corrupt. */ sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); p->nErr = 0; - if( nArg==1 ){ + if( zLike==0 ){ run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" @@ -3750,21 +3881,20 @@ static int do_meta_command(char *zLine, ShellState *p){ "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 ); }else{ - int i; - for(i=1; iwritableSchema ){ raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); @@ -4574,17 +4704,17 @@ static int do_meta_command(char *zLine, ShellState *p){ callback(&data, 1, new_argv, new_colv); rc = SQLITE_OK; }else{ - zShellStatic = azArg[1]; - rc = sqlite3_exec(p->db, + char *zSql; + zSql = sqlite3_mprintf( "SELECT sql FROM " " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE lower(tbl_name) LIKE shellstatic()" + "WHERE lower(tbl_name) LIKE %Q" " AND type!='meta' AND sql NOTNULL " - "ORDER BY rowid", - callback, &data, &zErrMsg); - zShellStatic = 0; + "ORDER BY rowid", azArg[1]); + rc = sqlite3_exec(p->db, zSql, callback, &data, &zErrMsg); + sqlite3_free(zSql); } }else if( nArg==1 ){ rc = sqlite3_exec(p->db, diff --git a/test/shell1.test b/test/shell1.test index 95c3130bdd..0912fa4240 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -300,7 +300,7 @@ do_test shell1-3.4.2 { do_test shell1-3.4.3 { # too many arguments catchcmd "test.db" ".dump FOO BAD" -} {1 {Usage: .dump ?LIKE-PATTERN?}} +} {1 {Usage: .dump ?--preserve-rowids? ?LIKE-PATTERN?}} # .echo ON|OFF Turn command echo on or off do_test shell1-3.5.1 { @@ -745,16 +745,58 @@ INSERT INTO "t1" VALUES(''); INSERT INTO "t1" VALUES(1); INSERT INTO "t1" VALUES(2.25); INSERT INTO "t1" VALUES('hello'); -INSERT INTO "t1" VALUES(X'807F'); +INSERT INTO "t1" VALUES(X'807f'); CREATE TABLE t3(x,y); INSERT INTO "t3" VALUES(1,NULL); INSERT INTO "t3" VALUES(2,''); INSERT INTO "t3" VALUES(3,1); INSERT INTO "t3" VALUES(4,2.25); INSERT INTO "t3" VALUES(5,'hello'); -INSERT INTO "t3" VALUES(6,X'807F'); +INSERT INTO "t3" VALUES(6,X'807f'); COMMIT;}} +do_test shell1-4.1.1 { + catchcmd test.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(x); +INSERT INTO "t1"(rowid,"x") VALUES(1,NULL); +INSERT INTO "t1"(rowid,"x") VALUES(2,''); +INSERT INTO "t1"(rowid,"x") VALUES(3,1); +INSERT INTO "t1"(rowid,"x") VALUES(4,2.25); +INSERT INTO "t1"(rowid,"x") VALUES(5,'hello'); +INSERT INTO "t1"(rowid,"x") VALUES(6,X'807f'); +CREATE TABLE t3(x,y); +INSERT INTO "t3"(rowid,"x","y") VALUES(1,1,NULL); +INSERT INTO "t3"(rowid,"x","y") VALUES(2,2,''); +INSERT INTO "t3"(rowid,"x","y") VALUES(3,3,1); +INSERT INTO "t3"(rowid,"x","y") VALUES(4,4,2.25); +INSERT INTO "t3"(rowid,"x","y") VALUES(5,5,'hello'); +INSERT INTO "t3"(rowid,"x","y") VALUES(6,6,X'807f'); +COMMIT;}} + +do_test shell1-4.1.2 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(1,null), (2,''), (3,1), + (4,2.25), (5,'hello'), (6,x'807f'); + } + catchcmd test2.db {.dump --preserve-rowids} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(x INTEGER PRIMARY KEY, y); +INSERT INTO "t1" VALUES(1,NULL); +INSERT INTO "t1" VALUES(2,''); +INSERT INTO "t1" VALUES(3,1); +INSERT INTO "t1" VALUES(4,2.25); +INSERT INTO "t1" VALUES(5,'hello'); +INSERT INTO "t1" VALUES(6,X'807f'); +COMMIT;}} + + # Test the output of ".mode insert" # do_test shell1-4.2.1 {