Improvements to the CSV virtual table.
FossilOrigin-Name: 0406ecbbe75513e342040b71fdd342462222dbb3820486b5f745d7865805c00b
This commit is contained in:
drh
parent
23bfa80c19
commit
6f147c54ef
131
ext/misc/csv.c
131
ext/misc/csv.c
@ -19,9 +19,9 @@
|
||||
** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
|
||||
** SELECT * FROM csv;
|
||||
**
|
||||
** The columns are named "c1", "c2", "c3", ... by default. But the
|
||||
** application can define its own CREATE TABLE statement as an additional
|
||||
** parameter. For example:
|
||||
** The columns are named "c1", "c2", "c3", ... by default. Or the
|
||||
** application can define its own CREATE TABLE statement using the
|
||||
** schema= parameter, like this:
|
||||
**
|
||||
** CREATE VIRTUAL TABLE temp.csv2 USING csv(
|
||||
** filename = "../http.log",
|
||||
@ -32,9 +32,9 @@
|
||||
** the data= parameter.
|
||||
**
|
||||
** If the columns=N parameter is supplied, then the CSV file is assumed to have
|
||||
** N columns. If the columns parameter is omitted, the CSV file is opened
|
||||
** as soon as the virtual table is constructed and the first row of the CSV
|
||||
** is read in order to count the tables.
|
||||
** N columns. If both the columns= and schema= parameters are omitted, then
|
||||
** the number and names of the columns is determined by the first line of
|
||||
** the CSV input.
|
||||
**
|
||||
** Some extra debugging features (used for testing virtual tables) are available
|
||||
** if this module is compiled with -DSQLITE_TEST.
|
||||
@ -436,6 +436,34 @@ static int csv_boolean(const char *z){
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Check to see if the string is of the form: "TAG = BOOLEAN" or just "TAG".
|
||||
** If it is, set *pValue to be the value of the boolean ("true" if there is
|
||||
** not "= BOOLEAN" component) and return non-zero. If the input string
|
||||
** does not begin with TAG, return zero.
|
||||
*/
|
||||
static int csv_boolean_parameter(
|
||||
const char *zTag, /* Tag we are looking for */
|
||||
int nTag, /* Size of the tag in bytes */
|
||||
const char *z, /* Input parameter */
|
||||
int *pValue /* Write boolean value here */
|
||||
){
|
||||
int b;
|
||||
z = csv_skip_whitespace(z);
|
||||
if( strncmp(zTag, z, nTag)!=0 ) return 0;
|
||||
z = csv_skip_whitespace(z + nTag);
|
||||
if( z[0]==0 ){
|
||||
*pValue = 1;
|
||||
return 1;
|
||||
}
|
||||
if( z[0]!='=' ) return 0;
|
||||
z = csv_skip_whitespace(z+1);
|
||||
b = csv_boolean(z);
|
||||
if( b>=0 ){
|
||||
*pValue = b;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
** Parameters:
|
||||
@ -469,6 +497,7 @@ static int csvtabConnect(
|
||||
#ifdef SQLITE_TEST
|
||||
int tstFlags = 0; /* Value for testflags=N parameter */
|
||||
#endif
|
||||
int b; /* Value of a boolean parameter */
|
||||
int nCol = -99; /* Value of the columns= parameter */
|
||||
CsvReader sRdr; /* A CSV file reader used to store an error
|
||||
** message and/or to count the number of columns */
|
||||
@ -493,21 +522,12 @@ static int csvtabConnect(
|
||||
if( j<sizeof(azParam)/sizeof(azParam[0]) ){
|
||||
if( sRdr.zErr[0] ) goto csvtab_connect_error;
|
||||
}else
|
||||
if( (zValue = csv_parameter("header",6,z))!=0 ){
|
||||
int x;
|
||||
if( csv_boolean_parameter("header",6,z,&b) ){
|
||||
if( bHeader>=0 ){
|
||||
csv_errmsg(&sRdr, "more than one 'header' parameter");
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
x = csv_boolean(zValue);
|
||||
if( x==1 ){
|
||||
bHeader = 1;
|
||||
}else if( x==0 ){
|
||||
bHeader = 0;
|
||||
}else{
|
||||
csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
bHeader = b;
|
||||
}else
|
||||
#ifdef SQLITE_TEST
|
||||
if( (zValue = csv_parameter("testflags",9,z))!=0 ){
|
||||
@ -521,53 +541,94 @@ static int csvtabConnect(
|
||||
}
|
||||
nCol = atoi(zValue);
|
||||
if( nCol<=0 ){
|
||||
csv_errmsg(&sRdr, "must have at least one column");
|
||||
csv_errmsg(&sRdr, "column= value must be positive");
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
}else
|
||||
{
|
||||
csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
|
||||
csv_errmsg(&sRdr, "bad parameter: '%s'", z);
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
}
|
||||
if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
|
||||
csv_errmsg(&sRdr, "must either filename= or data= but not both");
|
||||
csv_errmsg(&sRdr, "must specify either filename= or data= but not both");
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
|
||||
|
||||
if( (nCol<=0 || bHeader==1)
|
||||
&& csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA)
|
||||
){
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
pNew = sqlite3_malloc( sizeof(*pNew) );
|
||||
*ppVtab = (sqlite3_vtab*)pNew;
|
||||
if( pNew==0 ) goto csvtab_connect_oom;
|
||||
memset(pNew, 0, sizeof(*pNew));
|
||||
if( nCol>0 ){
|
||||
if( CSV_SCHEMA==0 ){
|
||||
sqlite3_str *pStr = sqlite3_str_new(0);
|
||||
char *zSep = "";
|
||||
int iCol = 0;
|
||||
sqlite3_str_appendf(pStr, "CREATE TABLE x(");
|
||||
if( nCol<0 && bHeader<1 ){
|
||||
nCol = 0;
|
||||
do{
|
||||
csv_read_one_field(&sRdr);
|
||||
nCol++;
|
||||
}while( sRdr.cTerm==',' );
|
||||
}
|
||||
if( nCol>0 && bHeader<1 ){
|
||||
for(iCol=0; iCol<nCol; iCol++){
|
||||
sqlite3_str_appendf(pStr, "%sc%d TEXT", zSep, iCol);
|
||||
zSep = ",";
|
||||
}
|
||||
}else{
|
||||
do{
|
||||
char *z = csv_read_one_field(&sRdr);
|
||||
if( (nCol>0 && iCol<nCol) || (nCol<0 && bHeader) ){
|
||||
sqlite3_str_appendf(pStr,"%s\"%w\" TEXT", zSep, z);
|
||||
zSep = ",";
|
||||
iCol++;
|
||||
}
|
||||
}while( sRdr.cTerm==',' );
|
||||
if( nCol<0 ){
|
||||
nCol = iCol;
|
||||
}else{
|
||||
while( iCol<nCol ){
|
||||
sqlite3_str_appendf(pStr,"%sc%d TEXT", zSep, ++iCol);
|
||||
zSep = ",";
|
||||
}
|
||||
}
|
||||
}
|
||||
pNew->nCol = nCol;
|
||||
}else{
|
||||
sqlite3_str_appendf(pStr, ")");
|
||||
CSV_SCHEMA = sqlite3_str_finish(pStr);
|
||||
if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
|
||||
}else if( nCol<0 ){
|
||||
do{
|
||||
csv_read_one_field(&sRdr);
|
||||
pNew->nCol++;
|
||||
}while( sRdr.cTerm==',' );
|
||||
}else{
|
||||
pNew->nCol = nCol;
|
||||
}
|
||||
pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0;
|
||||
pNew->zData = CSV_DATA; CSV_DATA = 0;
|
||||
#ifdef SQLITE_TEST
|
||||
pNew->tstFlags = tstFlags;
|
||||
#endif
|
||||
pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
|
||||
csv_reader_reset(&sRdr);
|
||||
if( CSV_SCHEMA==0 ){
|
||||
char *zSep = "";
|
||||
CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
|
||||
if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
|
||||
for(i=0; i<pNew->nCol; i++){
|
||||
CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
|
||||
zSep = ",";
|
||||
}
|
||||
CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
|
||||
if( bHeader!=1 ){
|
||||
pNew->iStart = 0;
|
||||
}else if( pNew->zData ){
|
||||
pNew->iStart = (int)sRdr.iIn;
|
||||
}else{
|
||||
pNew->iStart = ftell(sRdr.in);
|
||||
}
|
||||
csv_reader_reset(&sRdr);
|
||||
rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
|
||||
if( rc ) goto csvtab_connect_error;
|
||||
if( rc ){
|
||||
csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db));
|
||||
goto csvtab_connect_error;
|
||||
}
|
||||
for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){
|
||||
sqlite3_free(azPValue[i]);
|
||||
}
|
||||
|
||||
16
manifest
16
manifest
@ -1,5 +1,5 @@
|
||||
C Fix\stypos\sin\sthe\sprevious\scheck-in.
|
||||
D 2018-11-15T19:12:22.983
|
||||
C Improvements\sto\sthe\sCSV\svirtual\stable.
|
||||
D 2018-11-16T01:42:26.183
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F Makefile.in b730006b54c990461d864c5387f2e6f13aadb0236804555fb010ed6865a5f058
|
||||
@ -278,7 +278,7 @@ F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c
|
||||
F ext/misc/closure.c 9f8fa11aa6c6e2f6d7296ffa88f103df4b46abd9602bcab3ea2f8fc24f334f63
|
||||
F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f
|
||||
F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189
|
||||
F ext/misc/csv.c 65297bcce8d5acd5aadef42acbe739aef5a2ef5e74c7b73361ca19f3e21de657
|
||||
F ext/misc/csv.c 88333dc9f7dcf6a8148406f10ae04261e24e3b4c721550ae33e9e71f1265c1f1
|
||||
F ext/misc/dbdump.c 12389a10c410fadf1e68eeb382def92d5a7fa9ce7cce4fb86a736fa2bac1000a
|
||||
F ext/misc/eval.c 6ea9b22a5fa0dd973b67ca4e53555be177bc0b7b263aadf1024429457c82c0e3
|
||||
F ext/misc/explain.c c82dd86f1156d32b284e0523a4bf6a93a85ab2a812caed48963e0774f3327185
|
||||
@ -764,7 +764,7 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
|
||||
F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
|
||||
F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
|
||||
F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c
|
||||
F test/csv01.test 6e1445b3207d574cff22fc41a8e549dfcf2466ee90546ada97d22a90fa89eb58
|
||||
F test/csv01.test 4a92840619ef435b905e6d3f35cd0644df23225d7b7967d7940b40f06d6a90a6
|
||||
F test/ctime.test 78749e6c9a5f0010d67985be80788f841e3cd2da18114e2ed6010399a7d807f3
|
||||
F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856
|
||||
F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f
|
||||
@ -1778,7 +1778,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 3d947e1ec0f0800fcad3a04e4876d5247fd1b23e51ed7f49d428fff6683e8a16
|
||||
R 0ba578f427646842341f9570a7cd07b6
|
||||
U mistachkin
|
||||
Z 18affa3b55ac4767cecb124c8a07bfe3
|
||||
P 9b37bbf5f338dea962331e4fd73dfa0baae98ba63344a2fa737b845b90ecb0c5
|
||||
R 357cf9494dad6c6e818e36c733fa5cbf
|
||||
U drh
|
||||
Z cada0122630474b8bae10d99d99df1dd
|
||||
|
||||
@ -1 +1 @@
|
||||
9b37bbf5f338dea962331e4fd73dfa0baae98ba63344a2fa737b845b90ecb0c5
|
||||
0406ecbbe75513e342040b71fdd342462222dbb3820486b5f745d7865805c00b
|
||||
@ -38,6 +38,63 @@ do_execsql_test 1.2 {
|
||||
SELECT rowid FROM t1;
|
||||
} {1 2 3 4}
|
||||
|
||||
do_execsql_test 1.3 {
|
||||
DROP TABLE temp.t1;
|
||||
CREATE VIRTUAL TABLE temp.t1 USING csv(
|
||||
data=
|
||||
'a,b,"mix-bloom-eel","soft opinion"
|
||||
1,2,3,4
|
||||
5,6,7,8
|
||||
9,10,11,12
|
||||
13,14,15,16
|
||||
',
|
||||
header=1
|
||||
);
|
||||
SELECT * FROM t1 WHERE "soft opinion"=12;
|
||||
} {9 10 11 12}
|
||||
do_execsql_test 1.4 {
|
||||
SELECT name FROM pragma_table_xinfo('t1');
|
||||
} {a b mix-bloom-eel {soft opinion}}
|
||||
|
||||
do_execsql_test 1.5 {
|
||||
DROP TABLE temp.t1;
|
||||
CREATE VIRTUAL TABLE temp.t1 USING csv(
|
||||
data=
|
||||
'a,b,"mix-bloom-eel","soft opinion"
|
||||
1,2,3,4
|
||||
5,6,7,8
|
||||
9,10,11,12
|
||||
13,14,15,16
|
||||
',
|
||||
header=false
|
||||
);
|
||||
SELECT * FROM t1 WHERE c1='b';
|
||||
} {a b mix-bloom-eel {soft opinion}}
|
||||
do_execsql_test 1.6 {
|
||||
SELECT name FROM pragma_table_xinfo('t1');
|
||||
} {c0 c1 c2 c3}
|
||||
|
||||
do_execsql_test 1.7 {
|
||||
DROP TABLE temp.t1;
|
||||
CREATE VIRTUAL TABLE temp.t1 USING csv(
|
||||
data=
|
||||
'a,b,"mix-bloom-eel","soft opinion"
|
||||
1,2,3,4
|
||||
5,6,7,8
|
||||
9,10,11,12
|
||||
13,14,15,16
|
||||
',
|
||||
header,
|
||||
schema='CREATE TABLE x(x0,x1,x2,x3,x4)',
|
||||
columns=5
|
||||
);
|
||||
SELECT * FROM t1 WHERE x1='6';
|
||||
} {5 6 7 8 {}}
|
||||
do_execsql_test 1.8 {
|
||||
SELECT name FROM pragma_table_xinfo('t1');
|
||||
} {x0 x1 x2 x3 x4}
|
||||
|
||||
|
||||
do_execsql_test 2.0 {
|
||||
DROP TABLE t1;
|
||||
CREATE VIRTUAL TABLE temp.t2 USING csv(
|
||||
@ -107,7 +164,7 @@ do_catchsql_test 4.0 {
|
||||
'CREATE TABLE t3(a,b,c,d,PRIMARY KEY(a,b)) WITHOUT ROWID',
|
||||
testflags=1
|
||||
);
|
||||
} {1 {vtable constructor failed: t4}}
|
||||
} {1 {bad schema: 'CREATE TABLE t3(a,b,c,d,PRIMARY KEY(a,b)) WITHOUT ROWID' - not an error}}
|
||||
|
||||
# WITHOUT ROWID tables with a single-column PRIMARY KEY may be writable.
|
||||
do_catchsql_test 4.1 {
|
||||
@ -138,7 +195,7 @@ do_catchsql_test 4.2 {
|
||||
'CREATE TABLE t3(a,b,c,d) WITHOUT ROWID',
|
||||
testflags=1
|
||||
);
|
||||
} {1 {vtable constructor failed: t5}}
|
||||
} {1 {bad schema: 'CREATE TABLE t3(a,b,c,d) WITHOUT ROWID' - PRIMARY KEY missing on table t3}}
|
||||
|
||||
# 2018-04-24
|
||||
# Memory leak reported on the sqlite-users mailing list by Ralf Junker.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user