mirror of https://github.com/sqlite/sqlite
Update the ".import" command of the command-line shell so that it can
accept field values that span multiple lines and so that it issues error messages if the input text does not strictly conform to RFC4180. FossilOrigin-Name: 93f632152e464a89322a0130adaf9f342411bf7d
This commit is contained in:
drh
parent
cef4fb61f0
commit
db95f68b14
16
manifest
16
manifest
|
|
@ -1,5 +1,5 @@
|
|||
C Add\sthe\s"vtshim"\sextension,\simplementing\sa\swrapper\saround\sthe\svirtual\ntable\sinterface\sto\smake\sit\sDisposable\sfor\sthe\sCLR.\s\sNo\schanges\sto\sthe\ncore.
|
||||
D 2013-06-26T18:04:19.083
|
||||
C Update\sthe\s".import"\scommand\sof\sthe\scommand-line\sshell\sso\sthat\sit\scan\naccept\sfield\svalues\sthat\sspan\smultiple\slines\sand\sso\sthat\sit\sissues\nerror\smessages\sif\sthe\sinput\stext\sdoes\snot\sstrictly\sconform\sto\sRFC4180.
|
||||
D 2013-06-26T22:46:00.198
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
|
|
@ -217,7 +217,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
|
|||
F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8
|
||||
F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
|
||||
F src/select.c 91b62654caf8dfe292fb8882715e575d34ad3874
|
||||
F src/shell.c a02544af6697c5782d29ec3204616f35ed9e8458
|
||||
F src/shell.c 92cbe95eadc1c423422d36beac3609a9422889d1
|
||||
F src/sqlite.h.in 5f86553f4c1d8b4a9069285ed19e7981451ea77a
|
||||
F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
|
||||
F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5
|
||||
|
|
@ -773,11 +773,11 @@ F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21
|
|||
F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
|
||||
F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa
|
||||
F test/sharedlock.test 927a4b6da11978c82b857dbdb20a932aad732123
|
||||
F test/shell1.test 4a2f57952719972c6f862134463f8712e953c038
|
||||
F test/shell1.test 338f51e6ff543720c609178bda81c2606df8df8d
|
||||
F test/shell2.test 037d6ad16e873354195d30bb2dc4b5321788154a
|
||||
F test/shell3.test 9196c42772d575685e722c92b4b39053c6ebba59
|
||||
F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9
|
||||
F test/shell5.test fa2188bbb13fe2d183fd04a5f7b512650c35ef5d
|
||||
F test/shell5.test 0fed7823d57e80f79da2c5f350e50aa86011f24f
|
||||
F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
|
||||
F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868
|
||||
F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329
|
||||
|
|
@ -1098,7 +1098,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
|||
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
|
||||
F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
|
||||
F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
|
||||
P ebac5afa471526dffc8026e66753263476137a3b b4a0d5327addd90bef758e6a1403ac69f61b3886
|
||||
R a3237e173c634535eb67beba24018ada
|
||||
P 6c3839ef311a53076650c6479c932e545a26b96f
|
||||
R 20f28da1c568459e4d7e332d42754a7d
|
||||
U drh
|
||||
Z 356f790fa566cdf1861055fde4a499e2
|
||||
Z a56fe80d4e15f5a77335d76e21c16046
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
6c3839ef311a53076650c6479c932e545a26b96f
|
||||
93f632152e464a89322a0130adaf9f342411bf7d
|
||||
247
src/shell.c
247
src/shell.c
|
|
@ -1650,6 +1650,107 @@ static void test_breakpoint(void){
|
|||
nCall++;
|
||||
}
|
||||
|
||||
/*
|
||||
** An object used to read a CSV file
|
||||
*/
|
||||
typedef struct CSVReader CSVReader;
|
||||
struct CSVReader {
|
||||
const char *zFile; /* Name of the input file */
|
||||
FILE *in; /* Read the CSV text from this input stream */
|
||||
char *z; /* Accumulated text for a field */
|
||||
int n; /* Number of bytes in z */
|
||||
int nAlloc; /* Space allocated for z[] */
|
||||
int nLine; /* Current line number */
|
||||
int cTerm; /* Character that terminated the most recent field */
|
||||
int cSeparator; /* The separator character. (Usually ",") */
|
||||
};
|
||||
|
||||
/* Append a single byte to z[] */
|
||||
static void csv_append_char(CSVReader *p, int c){
|
||||
if( p->n+1>=p->nAlloc ){
|
||||
p->nAlloc += p->nAlloc + 100;
|
||||
p->z = sqlite3_realloc(p->z, p->nAlloc);
|
||||
if( p->z==0 ){
|
||||
fprintf(stderr, "out of memory\n");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
p->z[p->n++] = (char)c;
|
||||
}
|
||||
|
||||
/* Read a single field of CSV text. Compatible with rfc4180 and extended
|
||||
** with the option of having a separator other than ",".
|
||||
**
|
||||
** + Input comes from p->in.
|
||||
** + Store results in p->z of length p->n. Space to hold p->z comes
|
||||
** from sqlite3_malloc().
|
||||
** + Use p->cSep as the separator. The default is ",".
|
||||
** + Keep track of the line number in p->nLine.
|
||||
** + Store the character that terminates the field in p->cTerm. Store
|
||||
** EOF on end-of-file.
|
||||
** + Report syntax errors on stderr
|
||||
*/
|
||||
static char *csv_read_one_field(CSVReader *p){
|
||||
int c, pc;
|
||||
int cSep = p->cSeparator;
|
||||
p->n = 0;
|
||||
c = fgetc(p->in);
|
||||
if( c==EOF || seenInterrupt ){
|
||||
p->cTerm = EOF;
|
||||
return 0;
|
||||
}
|
||||
if( c=='"' ){
|
||||
int startLine = p->nLine;
|
||||
int cQuote = c;
|
||||
pc = 0;
|
||||
while( 1 ){
|
||||
c = fgetc(p->in);
|
||||
if( c=='\n' ) p->nLine++;
|
||||
if( c==cQuote ){
|
||||
if( pc==cQuote ){
|
||||
pc = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if( (c==cSep && pc==cQuote)
|
||||
|| (c=='\n' && pc==cQuote)
|
||||
|| (c=='\n' && pc=='\r' && p->n>2 && p->z[p->n-2]==cQuote)
|
||||
|| (c==EOF && pc==cQuote)
|
||||
){
|
||||
do{ p->n--; }while( p->z[p->n]!=cQuote );
|
||||
p->z[p->n] = 0;
|
||||
p->cTerm = c;
|
||||
break;
|
||||
}
|
||||
if( pc==cQuote && c!='\r' ){
|
||||
fprintf(stderr, "%s:%d: unescaped %c character\n",
|
||||
p->zFile, p->nLine, cQuote);
|
||||
}
|
||||
if( c==EOF ){
|
||||
fprintf(stderr, "%s:%d: unterminated %c-quoted field\n",
|
||||
p->zFile, startLine, cQuote);
|
||||
p->z[p->n] = 0;
|
||||
p->cTerm = EOF;
|
||||
break;
|
||||
}
|
||||
csv_append_char(p, c);
|
||||
pc = c;
|
||||
}
|
||||
}else{
|
||||
csv_append_char(p, c);
|
||||
while( (c = fgetc(p->in))!=EOF && c!=cSep && c!='\n' ){
|
||||
csv_append_char(p, c);
|
||||
}
|
||||
if( c=='\n' ){
|
||||
p->nLine++;
|
||||
if( p->n>1 && p->z[p->n-1]=='\r' ) p->n--;
|
||||
}
|
||||
p->z[p->n] = 0;
|
||||
p->cTerm = c;
|
||||
}
|
||||
return p->z;
|
||||
}
|
||||
|
||||
/*
|
||||
** If an input line begins with "." then invoke this routine to
|
||||
** process that line.
|
||||
|
|
@ -1888,48 +1989,81 @@ static int do_meta_command(char *zLine, struct callback_data *p){
|
|||
|
||||
if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg==3 ){
|
||||
char *zTable = azArg[2]; /* Insert data into this table */
|
||||
char *zFile = azArg[1]; /* The file from which to extract data */
|
||||
sqlite3_stmt *pStmt = NULL; /* A statement */
|
||||
int nCol; /* Number of columns in the table */
|
||||
int nByte; /* Number of bytes in an SQL string */
|
||||
int i, j; /* Loop counters */
|
||||
int nSep; /* Number of bytes in p->separator[] */
|
||||
char *zSql; /* An SQL statement */
|
||||
char *zLine; /* A single line of input from the file */
|
||||
char **azCol; /* zLine[] broken up into columns */
|
||||
char *zCommit; /* How to commit changes */
|
||||
FILE *in; /* The input file */
|
||||
int lineno = 0; /* Line number of input file */
|
||||
CSVReader sCsv; /* Reader context */
|
||||
|
||||
seenInterrupt = 0;
|
||||
memset(&sCsv, 0, sizeof(sCsv));
|
||||
sCsv.zFile = azArg[1];
|
||||
sCsv.nLine = 1;
|
||||
open_db(p);
|
||||
nSep = strlen30(p->separator);
|
||||
if( nSep==0 ){
|
||||
fprintf(stderr, "Error: non-null separator required for import\n");
|
||||
return 1;
|
||||
}
|
||||
if( nSep>1 ){
|
||||
fprintf(stderr, "Error: multi-character separators not allowed"
|
||||
" for import\n");
|
||||
return 1;
|
||||
}
|
||||
sCsv.in = fopen(sCsv.zFile, "rb");
|
||||
if( sCsv.in==0 ){
|
||||
fprintf(stderr, "Error: cannot open \"%s\"\n", sCsv.zFile);
|
||||
return 1;
|
||||
}
|
||||
sCsv.cSeparator = p->separator[0];
|
||||
zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
|
||||
if( zSql==0 ){
|
||||
fprintf(stderr, "Error: out of memory\n");
|
||||
fclose(sCsv.in);
|
||||
return 1;
|
||||
}
|
||||
nByte = strlen30(zSql);
|
||||
rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
|
||||
if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(db))==0 ){
|
||||
char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
|
||||
char cSep = '(';
|
||||
while( csv_read_one_field(&sCsv) ){
|
||||
zCreate = sqlite3_mprintf("%z%c\n \"%s\" TEXT", zCreate, cSep, sCsv.z);
|
||||
cSep = ',';
|
||||
if( sCsv.cTerm!=sCsv.cSeparator ) break;
|
||||
}
|
||||
zCreate = sqlite3_mprintf("%z\n)", zCreate);
|
||||
rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
|
||||
sqlite3_free(zCreate);
|
||||
if( rc ){
|
||||
fprintf(stderr, "CREATE TABLE %s(...) failed: %s\n", zTable,
|
||||
sqlite3_errmsg(db));
|
||||
sqlite3_free(sCsv.z);
|
||||
fclose(sCsv.in);
|
||||
return 1;
|
||||
}
|
||||
rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
|
||||
}
|
||||
sqlite3_free(zSql);
|
||||
if( rc ){
|
||||
if (pStmt) sqlite3_finalize(pStmt);
|
||||
fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db));
|
||||
fclose(sCsv.in);
|
||||
return 1;
|
||||
}
|
||||
nCol = sqlite3_column_count(pStmt);
|
||||
sqlite3_finalize(pStmt);
|
||||
pStmt = 0;
|
||||
if( nCol==0 ) return 0; /* no columns, no error */
|
||||
zSql = malloc( nByte + 20 + nCol*2 );
|
||||
zSql = sqlite3_malloc( nByte*2 + 20 + nCol*2 );
|
||||
if( zSql==0 ){
|
||||
fprintf(stderr, "Error: out of memory\n");
|
||||
fclose(sCsv.in);
|
||||
return 1;
|
||||
}
|
||||
sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zTable);
|
||||
sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
|
||||
j = strlen30(zSql);
|
||||
for(i=1; i<nCol; i++){
|
||||
zSql[j++] = ',';
|
||||
|
|
@ -1938,79 +2072,50 @@ static int do_meta_command(char *zLine, struct callback_data *p){
|
|||
zSql[j++] = ')';
|
||||
zSql[j] = 0;
|
||||
rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
|
||||
free(zSql);
|
||||
sqlite3_free(zSql);
|
||||
if( rc ){
|
||||
fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db));
|
||||
if (pStmt) sqlite3_finalize(pStmt);
|
||||
fclose(sCsv.in);
|
||||
return 1;
|
||||
}
|
||||
in = fopen(zFile, "rb");
|
||||
if( in==0 ){
|
||||
fprintf(stderr, "Error: cannot open \"%s\"\n", zFile);
|
||||
sqlite3_finalize(pStmt);
|
||||
return 1;
|
||||
}
|
||||
azCol = malloc( sizeof(azCol[0])*(nCol+1) );
|
||||
if( azCol==0 ){
|
||||
fprintf(stderr, "Error: out of memory\n");
|
||||
fclose(in);
|
||||
sqlite3_finalize(pStmt);
|
||||
return 1;
|
||||
}
|
||||
sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
|
||||
zCommit = "COMMIT";
|
||||
while( (zLine = local_getline(0, in, 1))!=0 ){
|
||||
char *z, c;
|
||||
int inQuote = 0;
|
||||
lineno++;
|
||||
azCol[0] = zLine;
|
||||
for(i=0, z=zLine; (c = *z)!=0; z++){
|
||||
if( c=='"' ) inQuote = !inQuote;
|
||||
if( c=='\n' ) lineno++;
|
||||
if( !inQuote && c==p->separator[0] && strncmp(z,p->separator,nSep)==0 ){
|
||||
*z = 0;
|
||||
i++;
|
||||
if( i<nCol ){
|
||||
azCol[i] = &z[nSep];
|
||||
z += nSep-1;
|
||||
}
|
||||
}
|
||||
} /* end for */
|
||||
*z = 0;
|
||||
if( i+1!=nCol ){
|
||||
fprintf(stderr,
|
||||
"Error: %s line %d: expected %d columns of data but found %d\n",
|
||||
zFile, lineno, nCol, i+1);
|
||||
zCommit = "ROLLBACK";
|
||||
free(zLine);
|
||||
rc = 1;
|
||||
break; /* from while */
|
||||
}
|
||||
do{
|
||||
int startLine = sCsv.nLine;
|
||||
for(i=0; i<nCol; i++){
|
||||
if( azCol[i][0]=='"' ){
|
||||
int k;
|
||||
for(z=azCol[i], j=1, k=0; z[j]; j++){
|
||||
if( z[j]=='"' ){ j++; if( z[j]==0 ) break; }
|
||||
z[k++] = z[j];
|
||||
}
|
||||
z[k] = 0;
|
||||
char *z = csv_read_one_field(&sCsv);
|
||||
if( z==0 && i==0 ) break;
|
||||
sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
|
||||
if( i<nCol-1 && sCsv.cTerm!=sCsv.cSeparator ){
|
||||
fprintf(stderr, "%s:%d: expected %d columns but found %d - "
|
||||
"filling the rest with NULL\n",
|
||||
sCsv.zFile, startLine, nCol, i+1);
|
||||
i++;
|
||||
while( i<nCol ){ sqlite3_bind_null(pStmt, i); i++; }
|
||||
}
|
||||
sqlite3_bind_text(pStmt, i+1, azCol[i], -1, SQLITE_STATIC);
|
||||
}
|
||||
sqlite3_step(pStmt);
|
||||
rc = sqlite3_reset(pStmt);
|
||||
free(zLine);
|
||||
if( rc!=SQLITE_OK ){
|
||||
fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db));
|
||||
zCommit = "ROLLBACK";
|
||||
rc = 1;
|
||||
break; /* from while */
|
||||
if( sCsv.cTerm==sCsv.cSeparator ){
|
||||
do{
|
||||
csv_read_one_field(&sCsv);
|
||||
i++;
|
||||
}while( sCsv.cTerm==sCsv.cSeparator );
|
||||
fprintf(stderr, "%s:%d: expected %d columns but found %d - "
|
||||
"extras ignored\n",
|
||||
sCsv.zFile, startLine, nCol, i);
|
||||
}
|
||||
} /* end while */
|
||||
free(azCol);
|
||||
fclose(in);
|
||||
if( i>=nCol ){
|
||||
sqlite3_step(pStmt);
|
||||
rc = sqlite3_reset(pStmt);
|
||||
if( rc!=SQLITE_OK ){
|
||||
fprintf(stderr, "%s:%d: INSERT failed: %s\n", sCsv.zFile, startLine,
|
||||
sqlite3_errmsg(db));
|
||||
}
|
||||
}
|
||||
}while( sCsv.cTerm!=EOF );
|
||||
|
||||
fclose(sCsv.in);
|
||||
sqlite3_free(sCsv.z);
|
||||
sqlite3_finalize(pStmt);
|
||||
sqlite3_exec(p->db, zCommit, 0, 0, 0);
|
||||
sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
|
||||
}else
|
||||
|
||||
if( c=='i' && strncmp(azArg[0], "indices", n)==0 && nArg<3 ){
|
||||
|
|
|
|||
|
|
@ -397,9 +397,9 @@ do_test shell1-3.11.1 {
|
|||
do_test shell1-3.11.2 {
|
||||
catchcmd "test.db" ".import FOO"
|
||||
} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}}
|
||||
do_test shell1-3.11.2 {
|
||||
catchcmd "test.db" ".import FOO BAR"
|
||||
} {1 {Error: no such table: BAR}}
|
||||
#do_test shell1-3.11.2 {
|
||||
# catchcmd "test.db" ".import FOO BAR"
|
||||
#} {1 {Error: no such table: BAR}}
|
||||
do_test shell1-3.11.3 {
|
||||
# too many arguments
|
||||
catchcmd "test.db" ".import FOO BAR BAD"
|
||||
|
|
|
|||
|
|
@ -45,9 +45,9 @@ do_test shell5-1.1.1 {
|
|||
do_test shell5-1.1.2 {
|
||||
catchcmd "test.db" ".import FOO"
|
||||
} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}}
|
||||
do_test shell5-1.1.2 {
|
||||
catchcmd "test.db" ".import FOO BAR"
|
||||
} {1 {Error: no such table: BAR}}
|
||||
#do_test shell5-1.1.2 {
|
||||
# catchcmd "test.db" ".import FOO BAR"
|
||||
#} {1 {Error: no such table: BAR}}
|
||||
do_test shell5-1.1.3 {
|
||||
# too many arguments
|
||||
catchcmd "test.db" ".import FOO BAR BAD"
|
||||
|
|
@ -101,7 +101,7 @@ do_test shell5-1.4.3 {
|
|||
puts $in "1"
|
||||
close $in
|
||||
set res [catchcmd "test.db" {.import shell5.csv t1}]
|
||||
} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}}
|
||||
} {1 {shell5.csv:1: expected 2 columns but found 1 - filling the rest with NULL}}
|
||||
|
||||
# import file with 1 row, 3 columns (expecting 2 cols)
|
||||
do_test shell5-1.4.4 {
|
||||
|
|
@ -109,14 +109,15 @@ do_test shell5-1.4.4 {
|
|||
puts $in "1|2|3"
|
||||
close $in
|
||||
set res [catchcmd "test.db" {.import shell5.csv t1}]
|
||||
} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}}
|
||||
} {1 {shell5.csv:1: expected 2 columns but found 3 - extras ignored}}
|
||||
|
||||
# import file with 1 row, 2 columns
|
||||
do_test shell5-1.4.5 {
|
||||
set in [open shell5.csv w]
|
||||
puts $in "1|2"
|
||||
close $in
|
||||
set res [catchcmd "test.db" {.import shell5.csv t1
|
||||
set res [catchcmd "test.db" {DELETE FROM t1;
|
||||
.import shell5.csv t1
|
||||
SELECT COUNT(*) FROM t1;}]
|
||||
} {0 1}
|
||||
|
||||
|
|
@ -197,15 +198,15 @@ SELECT length(b) FROM t1 WHERE a='8';}]
|
|||
# This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999.
|
||||
set cols 999
|
||||
do_test shell5-1.6.1 {
|
||||
set sql {CREATE TABLE t2(}
|
||||
set data {}
|
||||
for {set i 1} {$i<$cols} {incr i} {
|
||||
append sql "c$i,"
|
||||
append data "c$i|"
|
||||
}
|
||||
append data "c$cols\n";
|
||||
for {set i 1} {$i<$cols} {incr i} {
|
||||
append data "$i|"
|
||||
}
|
||||
append sql "c$cols);"
|
||||
append data "$cols"
|
||||
catchcmd "test.db" $sql
|
||||
set in [open shell5.csv w]
|
||||
puts $in $data
|
||||
close $in
|
||||
|
|
@ -214,14 +215,15 @@ SELECT COUNT(*) FROM t2;}]
|
|||
} {0 1}
|
||||
|
||||
# try importing a large number of rows
|
||||
set rows 999999
|
||||
set rows 99999
|
||||
do_test shell5-1.7.1 {
|
||||
set in [open shell5.csv w]
|
||||
puts $in a
|
||||
for {set i 1} {$i<=$rows} {incr i} {
|
||||
puts $in $i
|
||||
}
|
||||
close $in
|
||||
set res [catchcmd "test.db" {CREATE TABLE t3(a);
|
||||
set res [catchcmd "test.db" {.mode csv
|
||||
.import shell5.csv t3
|
||||
SELECT COUNT(*) FROM t3;}]
|
||||
} [list 0 $rows]
|
||||
|
|
|
|||
Loading…
Reference in New Issue