diff --git a/manifest b/manifest index 5872eec583..febb5a72c9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"vtshim"\sextension,\simplementing\sa\swrapper\saround\sthe\svirtual\ntable\sinterface\sto\smake\sit\sDisposable\sfor\sthe\sCLR.\s\sNo\schanges\sto\sthe\ncore. -D 2013-06-26T18:04:19.083 +C Update\sthe\s".import"\scommand\sof\sthe\scommand-line\sshell\sso\sthat\sit\scan\naccept\sfield\svalues\sthat\sspan\smultiple\slines\sand\sso\sthat\sit\sissues\nerror\smessages\sif\sthe\sinput\stext\sdoes\snot\sstrictly\sconform\sto\sRFC4180. +D 2013-06-26T22:46:00.198 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 91b62654caf8dfe292fb8882715e575d34ad3874 -F src/shell.c a02544af6697c5782d29ec3204616f35ed9e8458 +F src/shell.c 92cbe95eadc1c423422d36beac3609a9422889d1 F src/sqlite.h.in 5f86553f4c1d8b4a9069285ed19e7981451ea77a F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 @@ -773,11 +773,11 @@ F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa F test/sharedlock.test 927a4b6da11978c82b857dbdb20a932aad732123 -F test/shell1.test 4a2f57952719972c6f862134463f8712e953c038 +F test/shell1.test 338f51e6ff543720c609178bda81c2606df8df8d F test/shell2.test 037d6ad16e873354195d30bb2dc4b5321788154a F test/shell3.test 9196c42772d575685e722c92b4b39053c6ebba59 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9 -F test/shell5.test fa2188bbb13fe2d183fd04a5f7b512650c35ef5d +F test/shell5.test 0fed7823d57e80f79da2c5f350e50aa86011f24f F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 @@ -1098,7 +1098,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ebac5afa471526dffc8026e66753263476137a3b b4a0d5327addd90bef758e6a1403ac69f61b3886 -R a3237e173c634535eb67beba24018ada +P 6c3839ef311a53076650c6479c932e545a26b96f +R 20f28da1c568459e4d7e332d42754a7d U drh -Z 356f790fa566cdf1861055fde4a499e2 +Z a56fe80d4e15f5a77335d76e21c16046 diff --git a/manifest.uuid b/manifest.uuid index ac526c0c78..2ca371dd70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c3839ef311a53076650c6479c932e545a26b96f \ No newline at end of file +93f632152e464a89322a0130adaf9f342411bf7d \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 9a8a906944..e66125b23a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1650,6 +1650,107 @@ static void test_breakpoint(void){ nCall++; } +/* +** An object used to read a CSV file +*/ +typedef struct CSVReader CSVReader; +struct CSVReader { + const char *zFile; /* Name of the input file */ + FILE *in; /* Read the CSV text from this input stream */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + int cTerm; /* Character that terminated the most recent field */ + int cSeparator; /* The separator character. (Usually ",") */ +}; + +/* Append a single byte to z[] */ +static void csv_append_char(CSVReader *p, int c){ + if( p->n+1>=p->nAlloc ){ + p->nAlloc += p->nAlloc + 100; + p->z = sqlite3_realloc(p->z, p->nAlloc); + if( p->z==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + } + p->z[p->n++] = (char)c; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc(). +** + Use p->cSep as the separator. The default is ",". +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** + Report syntax errors on stderr +*/ +static char *csv_read_one_field(CSVReader *p){ + int c, pc; + int cSep = p->cSeparator; + p->n = 0; + c = fgetc(p->in); + if( c==EOF || seenInterrupt ){ + p->cTerm = EOF; + return 0; + } + if( c=='"' ){ + int startLine = p->nLine; + int cQuote = c; + pc = 0; + while( 1 ){ + c = fgetc(p->in); + if( c=='\n' ) p->nLine++; + if( c==cQuote ){ + if( pc==cQuote ){ + pc = 0; + continue; + } + } + if( (c==cSep && pc==cQuote) + || (c=='\n' && pc==cQuote) + || (c=='\n' && pc=='\r' && p->n>2 && p->z[p->n-2]==cQuote) + || (c==EOF && pc==cQuote) + ){ + do{ p->n--; }while( p->z[p->n]!=cQuote ); + p->z[p->n] = 0; + p->cTerm = c; + break; + } + if( pc==cQuote && c!='\r' ){ + fprintf(stderr, "%s:%d: unescaped %c character\n", + p->zFile, p->nLine, cQuote); + } + if( c==EOF ){ + fprintf(stderr, "%s:%d: unterminated %c-quoted field\n", + p->zFile, startLine, cQuote); + p->z[p->n] = 0; + p->cTerm = EOF; + break; + } + csv_append_char(p, c); + pc = c; + } + }else{ + csv_append_char(p, c); + while( (c = fgetc(p->in))!=EOF && c!=cSep && c!='\n' ){ + csv_append_char(p, c); + } + if( c=='\n' ){ + p->nLine++; + if( p->n>1 && p->z[p->n-1]=='\r' ) p->n--; + } + p->z[p->n] = 0; + p->cTerm = c; + } + return p->z; +} + /* ** If an input line begins with "." then invoke this routine to ** process that line. @@ -1888,48 +1989,81 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg==3 ){ char *zTable = azArg[2]; /* Insert data into this table */ - char *zFile = azArg[1]; /* The file from which to extract data */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ int nByte; /* Number of bytes in an SQL string */ int i, j; /* Loop counters */ int nSep; /* Number of bytes in p->separator[] */ char *zSql; /* An SQL statement */ - char *zLine; /* A single line of input from the file */ - char **azCol; /* zLine[] broken up into columns */ - char *zCommit; /* How to commit changes */ - FILE *in; /* The input file */ - int lineno = 0; /* Line number of input file */ + CSVReader sCsv; /* Reader context */ + seenInterrupt = 0; + memset(&sCsv, 0, sizeof(sCsv)); + sCsv.zFile = azArg[1]; + sCsv.nLine = 1; open_db(p); nSep = strlen30(p->separator); if( nSep==0 ){ fprintf(stderr, "Error: non-null separator required for import\n"); return 1; } + if( nSep>1 ){ + fprintf(stderr, "Error: multi-character separators not allowed" + " for import\n"); + return 1; + } + sCsv.in = fopen(sCsv.zFile, "rb"); + if( sCsv.in==0 ){ + fprintf(stderr, "Error: cannot open \"%s\"\n", sCsv.zFile); + return 1; + } + sCsv.cSeparator = p->separator[0]; zSql = sqlite3_mprintf("SELECT * FROM %s", zTable); if( zSql==0 ){ fprintf(stderr, "Error: out of memory\n"); + fclose(sCsv.in); return 1; } nByte = strlen30(zSql); rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(db))==0 ){ + char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable); + char cSep = '('; + while( csv_read_one_field(&sCsv) ){ + zCreate = sqlite3_mprintf("%z%c\n \"%s\" TEXT", zCreate, cSep, sCsv.z); + cSep = ','; + if( sCsv.cTerm!=sCsv.cSeparator ) break; + } + zCreate = sqlite3_mprintf("%z\n)", zCreate); + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc ){ + fprintf(stderr, "CREATE TABLE %s(...) failed: %s\n", zTable, + sqlite3_errmsg(db)); + sqlite3_free(sCsv.z); + fclose(sCsv.in); + return 1; + } + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + } sqlite3_free(zSql); if( rc ){ if (pStmt) sqlite3_finalize(pStmt); fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db)); + fclose(sCsv.in); return 1; } nCol = sqlite3_column_count(pStmt); sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ - zSql = malloc( nByte + 20 + nCol*2 ); + zSql = sqlite3_malloc( nByte*2 + 20 + nCol*2 ); if( zSql==0 ){ fprintf(stderr, "Error: out of memory\n"); + fclose(sCsv.in); return 1; } - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zTable); + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); j = strlen30(zSql); for(i=1; idb, zSql, -1, &pStmt, 0); - free(zSql); + sqlite3_free(zSql); if( rc ){ fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db)); if (pStmt) sqlite3_finalize(pStmt); + fclose(sCsv.in); return 1; } - in = fopen(zFile, "rb"); - if( in==0 ){ - fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); - sqlite3_finalize(pStmt); - return 1; - } - azCol = malloc( sizeof(azCol[0])*(nCol+1) ); - if( azCol==0 ){ - fprintf(stderr, "Error: out of memory\n"); - fclose(in); - sqlite3_finalize(pStmt); - return 1; - } - sqlite3_exec(p->db, "BEGIN", 0, 0, 0); - zCommit = "COMMIT"; - while( (zLine = local_getline(0, in, 1))!=0 ){ - char *z, c; - int inQuote = 0; - lineno++; - azCol[0] = zLine; - for(i=0, z=zLine; (c = *z)!=0; z++){ - if( c=='"' ) inQuote = !inQuote; - if( c=='\n' ) lineno++; - if( !inQuote && c==p->separator[0] && strncmp(z,p->separator,nSep)==0 ){ - *z = 0; - i++; - if( i=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "%s:%d: INSERT failed: %s\n", sCsv.zFile, startLine, + sqlite3_errmsg(db)); + } + } + }while( sCsv.cTerm!=EOF ); + + fclose(sCsv.in); + sqlite3_free(sCsv.z); sqlite3_finalize(pStmt); - sqlite3_exec(p->db, zCommit, 0, 0, 0); + sqlite3_exec(p->db, "COMMIT", 0, 0, 0); }else if( c=='i' && strncmp(azArg[0], "indices", n)==0 && nArg<3 ){ diff --git a/test/shell1.test b/test/shell1.test index c60f3af702..798a7e696f 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -397,9 +397,9 @@ do_test shell1-3.11.1 { do_test shell1-3.11.2 { catchcmd "test.db" ".import FOO" } {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell1-3.11.2 { - catchcmd "test.db" ".import FOO BAR" -} {1 {Error: no such table: BAR}} +#do_test shell1-3.11.2 { +# catchcmd "test.db" ".import FOO BAR" +#} {1 {Error: no such table: BAR}} do_test shell1-3.11.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" diff --git a/test/shell5.test b/test/shell5.test index d90cedf949..343e4cfcd1 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -45,9 +45,9 @@ do_test shell5-1.1.1 { do_test shell5-1.1.2 { catchcmd "test.db" ".import FOO" } {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell5-1.1.2 { - catchcmd "test.db" ".import FOO BAR" -} {1 {Error: no such table: BAR}} +#do_test shell5-1.1.2 { +# catchcmd "test.db" ".import FOO BAR" +#} {1 {Error: no such table: BAR}} do_test shell5-1.1.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" @@ -101,7 +101,7 @@ do_test shell5-1.4.3 { puts $in "1" close $in set res [catchcmd "test.db" {.import shell5.csv t1}] -} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}} +} {1 {shell5.csv:1: expected 2 columns but found 1 - filling the rest with NULL}} # import file with 1 row, 3 columns (expecting 2 cols) do_test shell5-1.4.4 { @@ -109,14 +109,15 @@ do_test shell5-1.4.4 { puts $in "1|2|3" close $in set res [catchcmd "test.db" {.import shell5.csv t1}] -} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}} +} {1 {shell5.csv:1: expected 2 columns but found 3 - extras ignored}} # import file with 1 row, 2 columns do_test shell5-1.4.5 { set in [open shell5.csv w] puts $in "1|2" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db" {DELETE FROM t1; +.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 1} @@ -197,15 +198,15 @@ SELECT length(b) FROM t1 WHERE a='8';}] # This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999. set cols 999 do_test shell5-1.6.1 { - set sql {CREATE TABLE t2(} set data {} for {set i 1} {$i<$cols} {incr i} { - append sql "c$i," + append data "c$i|" + } + append data "c$cols\n"; + for {set i 1} {$i<$cols} {incr i} { append data "$i|" } - append sql "c$cols);" append data "$cols" - catchcmd "test.db" $sql set in [open shell5.csv w] puts $in $data close $in @@ -214,14 +215,15 @@ SELECT COUNT(*) FROM t2;}] } {0 1} # try importing a large number of rows -set rows 999999 +set rows 99999 do_test shell5-1.7.1 { set in [open shell5.csv w] + puts $in a for {set i 1} {$i<=$rows} {incr i} { puts $in $i } close $in - set res [catchcmd "test.db" {CREATE TABLE t3(a); + set res [catchcmd "test.db" {.mode csv .import shell5.csv t3 SELECT COUNT(*) FROM t3;}] } [list 0 $rows]