mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhance the code in unionvtab.c to also provide the "swarmvtab" virtual table

Browse Source 
module. There are still several problems on this branch.

FossilOrigin-Name: 03d94388d62fd0f1fae377d273bbd5561208adc34bd97f7ce27783b30a369fd7
This commit is contained in:
dan 2017-08-02 19:59:56 +00:00
parent 754ee285fa
commit 63331b1af3
4 changed files with 327 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

276
ext/misc/unionvtab.c
View File

@ -65,6 +65,8 @@ SQLITE_EXTENSION_INIT1
# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
#endif
#define SWARMVTAB_MAX_ATTACHED 9
typedef struct UnionCsr UnionCsr;
typedef struct UnionTab UnionTab;
typedef struct UnionSrc UnionSrc;
@ -79,6 +81,11 @@ struct UnionSrc {
char *zTab; /* Source table name */
sqlite3_int64 iMin; /* Minimum rowid */
sqlite3_int64 iMax; /* Maximum rowid */
/* Fields used by swarmvtab only */
char *zFile; /* File to ATTACH */
int bAttached; /* True if currently attached */
UnionSrc *pNextAttached; /* Next in list of all attached sources */
};
/*
@ -87,9 +94,17 @@ struct UnionSrc {
struct UnionTab {
sqlite3_vtab base; /* Base class - must be first */
sqlite3 *db; /* Database handle */
int bSwarm; /* 1 for "swarmvtab", 0 for "unionvtab" */
int iPK; /* INTEGER PRIMARY KEY column, or -1 */
int nSrc; /* Number of elements in the aSrc[] array */
sqlite3_stmt *pSourceStr; /* Used by unionSourceToStr() */
UnionSrc *aSrc; /* Array of source tables, sorted by rowid */
/* Used by swarmvtab only */
char *zSourceStr; /* Expected unionSourceToStr() value */
UnionSrc *pAttached; /* First in list of attached sources */
int nAttach; /* Current number of attached sources */
int nMaxAttach; /* Maximum number of attached sources */
};
/*
@ -98,6 +113,10 @@ struct UnionTab {
struct UnionCsr {
sqlite3_vtab_cursor base; /* Base class - must be first */
sqlite3_stmt *pStmt; /* SQL statement to run */
/* Used by swarmvtab only */
sqlite3_int64 iMaxRowid; /* Last rowid to visit */
int iTab; /* Index of table read by pStmt */
};
/*
@ -204,7 +223,7 @@ static sqlite3_stmt *unionPrepare(
sqlite3_stmt *pRet = 0;
if( *pRc==SQLITE_OK ){
int rc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0);
if( rc!=SQLITE_OK ){
if( rc!=SQLITE_OK && pzErr ){
*pzErr = sqlite3_mprintf("sql error: %s", sqlite3_errmsg(db));
*pRc = rc;
}
@ -270,6 +289,31 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt){
if( *pRc==SQLITE_OK ) *pRc = rc;
}
static int unionDetachDatabase(UnionTab *pTab, char **pzErr){
sqlite3_stmt *pStmt = 0;
int rc = SQLITE_OK;
UnionSrc **pp;
assert( pTab->pAttached );
pp = &pTab->pAttached;
while( (*pp)->pNextAttached ){
pp = &(*pp)->pNextAttached;
}
pStmt = unionPreparePrintf(&rc, pzErr, pTab->db, "DETACH %s", (*pp)->zDb);
if( rc==SQLITE_OK ) sqlite3_step(pStmt);
unionFinalize(&rc, pStmt);
assert( rc==SQLITE_OK );
if( rc==SQLITE_OK ){
assert( (*pp)->bAttached && (*pp)->pNextAttached==0 );
(*pp)->bAttached = 0;
*pp = 0;
pTab->nAttach--;
}
return rc;
}
/*
** xDisconnect method.
*/
@ -280,13 +324,51 @@ static int unionDisconnect(sqlite3_vtab *pVtab){
for(i=0; i<pTab->nSrc; i++){
sqlite3_free(pTab->aSrc[i].zDb);
sqlite3_free(pTab->aSrc[i].zTab);
sqlite3_free(pTab->aSrc[i].zFile);
}
while( pTab->pAttached ){
if( unionDetachDatabase(pTab, 0)!=SQLITE_OK ) break;
}
sqlite3_finalize(pTab->pSourceStr);
sqlite3_free(pTab->zSourceStr);
sqlite3_free(pTab->aSrc);
sqlite3_free(pTab);
}
return SQLITE_OK;
}
/*
** Check that the table identified by pSrc is a rowid table. If not,
** return SQLITE_ERROR and set (*pzErr) to point to an English language
** error message. If the table is a rowid table and no error occurs,
** return SQLITE_OK and leave (*pzErr) unmodified.
*/
static int unionIsIntkeyTable(
sqlite3 *db, /* Database handle */
UnionSrc *pSrc, /* Source table to test */
char **pzErr /* OUT: Error message */
){
int bPk = 0;
const char *zType = 0;
int rc;
sqlite3_table_column_metadata(
db, pSrc->zDb, pSrc->zTab, "_rowid_", &zType, 0, 0, &bPk, 0
);
rc = sqlite3_errcode(db);
if( rc==SQLITE_ERROR
|| (rc==SQLITE_OK && (!bPk || sqlite3_stricmp("integer", zType)))
){
rc = SQLITE_ERROR;
*pzErr = sqlite3_mprintf("no such rowid table: %s%s%s",
(pSrc->zDb ? pSrc->zDb : ""),
(pSrc->zDb ? "." : ""),
pSrc->zTab
);
}
return rc;
}
/*
** This function is a no-op if *pRc is other than SQLITE_OK when it is
** called. In this case it returns NULL.
@ -306,41 +388,28 @@ static int unionDisconnect(sqlite3_vtab *pVtab){
*/
static char *unionSourceToStr(
int *pRc, /* IN/OUT: Error code */
sqlite3 *db, /* Database handle */
UnionTab *pTab, /* Virtual table object */
UnionSrc *pSrc, /* Source table to test */
sqlite3_stmt *pStmt,
char **pzErr /* OUT: Error message */
){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
int bPk = 0;
const char *zType = 0;
int rc;
sqlite3_table_column_metadata(
db, pSrc->zDb, pSrc->zTab, "_rowid_", &zType, 0, 0, &bPk, 0
);
rc = sqlite3_errcode(db);
if( rc==SQLITE_ERROR
|| (rc==SQLITE_OK && (!bPk || sqlite3_stricmp("integer", zType)))
){
rc = SQLITE_ERROR;
*pzErr = sqlite3_mprintf("no such rowid table: %s%s%s",
(pSrc->zDb ? pSrc->zDb : ""),
(pSrc->zDb ? "." : ""),
pSrc->zTab
int rc = unionIsIntkeyTable(pTab->db, pSrc, pzErr);
if( rc==SQLITE_OK && pTab->pSourceStr==0 ){
pTab->pSourceStr = unionPrepare(&rc, pTab->db,
"SELECT group_concat(quote(name) || '.' || quote(type)) "
"FROM pragma_table_info(?, ?)", pzErr
);
}
if( rc==SQLITE_OK ){
sqlite3_bind_text(pStmt, 1, pSrc->zTab, -1, SQLITE_STATIC);
sqlite3_bind_text(pStmt, 2, pSrc->zDb, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
zRet = unionStrdup(&rc, (const char*)sqlite3_column_text(pStmt, 0));
sqlite3_bind_text(pTab->pSourceStr, 1, pSrc->zTab, -1, SQLITE_STATIC);
sqlite3_bind_text(pTab->pSourceStr, 2, pSrc->zDb, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pTab->pSourceStr) ){
const char *z = (const char*)sqlite3_column_text(pTab->pSourceStr, 0);
zRet = unionStrdup(&rc, z);
}
unionReset(&rc, pStmt, pzErr);
unionReset(&rc, pTab->pSourceStr, pzErr);
}
*pRc = rc;
}
@ -356,25 +425,19 @@ static char *unionSourceToStr(
** other error occurs, SQLITE_OK is returned.
*/
static int unionSourceCheck(UnionTab *pTab, char **pzErr){
const char *zSql =
"SELECT group_concat(quote(name) || '.' || quote(type)) "
"FROM pragma_table_info(?, ?)";
int rc = SQLITE_OK;
assert( *pzErr==0 );
if( pTab->nSrc==0 ){
*pzErr = sqlite3_mprintf("no source tables configured");
rc = SQLITE_ERROR;
}else{
sqlite3_stmt *pStmt = 0;
char *z0 = 0;
int i;
pStmt = unionPrepare(&rc, pTab->db, zSql, pzErr);
if( rc==SQLITE_OK ){
z0 = unionSourceToStr(&rc, pTab->db, &pTab->aSrc[0], pStmt, pzErr);
}
z0 = unionSourceToStr(&rc, pTab, &pTab->aSrc[0], pzErr);
for(i=1; i<pTab->nSrc; i++){
char *z = unionSourceToStr(&rc, pTab->db, &pTab->aSrc[i], pStmt, pzErr);
char *z = unionSourceToStr(&rc, pTab, &pTab->aSrc[i], pzErr);
if( rc==SQLITE_OK && sqlite3_stricmp(z, z0) ){
*pzErr = sqlite3_mprintf("source table schema mismatch");
rc = SQLITE_ERROR;
@ -382,12 +445,58 @@ static int unionSourceCheck(UnionTab *pTab, char **pzErr){
sqlite3_free(z);
}
unionFinalize(&rc, pStmt);
unionFinalize(&rc, pTab->pSourceStr);
pTab->pSourceStr = 0;
sqlite3_free(z0);
}
return rc;
}
static int unionAttachDatabase(UnionTab *pTab, int iSrc, char **pzErr){
int rc = SQLITE_OK;
UnionSrc *pSrc = &pTab->aSrc[iSrc];
assert( pTab->bSwarm && iSrc<pTab->nSrc );
if( pSrc->bAttached==0 ){
sqlite3_stmt *pStmt;
if( pTab->nAttach>=pTab->nMaxAttach ){
rc = unionDetachDatabase(pTab, pzErr);
}
pStmt = unionPreparePrintf(
&rc, pzErr, pTab->db, "ATTACH %Q AS %s", pSrc->zFile, pSrc->zDb
);
if( rc==SQLITE_OK ){
sqlite3_step(pStmt);
rc = sqlite3_finalize(pStmt);
}
if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
}else{
char *z = unionSourceToStr(&rc, pTab, pSrc, pzErr);
if( rc==SQLITE_OK ){
if( pTab->zSourceStr==0 ){
pTab->zSourceStr = z;
}else{
if( sqlite3_stricmp(z, pTab->zSourceStr) ){
*pzErr = sqlite3_mprintf("source table schema mismatch");
rc = SQLITE_ERROR;
}
sqlite3_free(z);
}
}
}
pSrc->pNextAttached = pTab->pAttached;
pSrc->bAttached = 1;
pTab->pAttached = pSrc;
pTab->nAttach++;
}
return rc;
}
/*
** xConnect/xCreate method.
**
@ -407,14 +516,16 @@ static int unionConnect(
){
UnionTab *pTab = 0;
int rc = SQLITE_OK;
int bSwarm = (pAux==0 ? 0 : 1);
const char *zVtab = (bSwarm ? "swarmvtab" : "unionvtab");
const char *zName = argv[2];
(void)pAux; /* Suppress harmless 'unused parameter' warning */
if( sqlite3_stricmp("temp", argv[1]) ){
/* unionvtab tables may only be created in the temp schema */
*pzErr = sqlite3_mprintf("unionvtab tables must be created in TEMP schema");
*pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab);
rc = SQLITE_ERROR;
}else if( argc!=4 ){
*pzErr = sqlite3_mprintf("wrong number of arguments for unionvtab");
*pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab);
rc = SQLITE_ERROR;
}else{
int nAlloc = 0; /* Allocated size of pTab->aSrc[] */
@ -464,19 +575,44 @@ static int unionConnect(
rc = SQLITE_ERROR;
}
pSrc = &pTab->aSrc[pTab->nSrc++];
pSrc->zDb = unionStrdup(&rc, zDb);
pSrc->zTab = unionStrdup(&rc, zTab);
pSrc->iMin = iMin;
pSrc->iMax = iMax;
if( rc==SQLITE_OK ){
pSrc = &pTab->aSrc[pTab->nSrc++];
pSrc->zTab = unionStrdup(&rc, zTab);
pSrc->iMin = iMin;
pSrc->iMax = iMax;
if( bSwarm ){
pSrc->zFile = unionStrdup(&rc, zDb);
pSrc->zDb = sqlite3_mprintf("swm_%s_%d", zName, pTab->nSrc);
if( pSrc->zDb==0 ) rc = SQLITE_NOMEM;
}else{
pSrc->zDb = unionStrdup(&rc, zDb);
pSrc->bAttached = 1;
}
}
}
unionFinalize(&rc, pStmt);
pStmt = 0;
/* Verify that all source tables exist and have compatible schemas. */
/* It is an error if the SELECT statement returned zero rows. If only
** because there is no way to determine the schema of the virtual
** table in this case. */
if( rc==SQLITE_OK && pTab->nSrc==0 ){
*pzErr = sqlite3_mprintf("no source tables configured");
rc = SQLITE_ERROR;
}
/* For unionvtab, verify that all source tables exist and have
** compatible schemas. For swarmvtab, attach the first database and
** check that the first table is a rowid table only. */
if( rc==SQLITE_OK ){
pTab->db = db;
rc = unionSourceCheck(pTab, pzErr);
pTab->bSwarm = bSwarm;
pTab->nMaxAttach = SWARMVTAB_MAX_ATTACHED;
if( bSwarm ){
rc = unionAttachDatabase(pTab, 0, pzErr);
}else{
rc = unionSourceCheck(pTab, pzErr);
}
}
/* Compose a CREATE TABLE statement and pass it to declare_vtab() */
@ -535,16 +671,40 @@ static int unionClose(sqlite3_vtab_cursor *cur){
/*
** xNext
*/
static int unionNext(sqlite3_vtab_cursor *cur){
UnionCsr *pCsr = (UnionCsr*)cur;
int rc;
static int doUnionNext(UnionCsr *pCsr){
int rc = SQLITE_OK;
assert( pCsr->pStmt );
if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){
UnionTab *pTab = (UnionTab*)pCsr->base.pVtab;
rc = sqlite3_finalize(pCsr->pStmt);
pCsr->pStmt = 0;
}else{
rc = SQLITE_OK;
if( rc==SQLITE_OK && pTab->bSwarm ){
pCsr->iTab++;
if( pCsr->iTab<pTab->nSrc ){
UnionSrc *pSrc = &pTab->aSrc[pCsr->iTab];
if( pCsr->iMaxRowid>=pSrc->iMin ){
/* It is necessary to scan the next table. */
rc = unionAttachDatabase(pTab, pCsr->iTab, &pTab->base.zErrMsg);
pCsr->pStmt = unionPreparePrintf(&rc, &pTab->base.zErrMsg, pTab->db,
"SELECT rowid, * FROM %Q.%Q %s %lld",
pSrc->zDb, pSrc->zTab,
(pSrc->iMax>pCsr->iMaxRowid ? "WHERE _rowid_ <=" : "-- "),
pCsr->iMaxRowid
);
if( rc==SQLITE_OK ) rc = SQLITE_ROW;
}
}
}
}
return rc;
}
static int unionNext(sqlite3_vtab_cursor *cur){
int rc;
do {
rc = doUnionNext((UnionCsr*)cur);
}while( rc==SQLITE_ROW );
return rc;
}
@ -674,6 +834,13 @@ static int unionFilter(
zSql = sqlite3_mprintf("%z %s rowid<=%lld", zSql, zWhere, iMax);
}
}
if( pTab->bSwarm ){
pCsr->iTab = i;
pCsr->iMaxRowid = iMax;
rc = unionAttachDatabase(pTab, i, &pTab->base.zErrMsg);
break;
}
}
@ -791,8 +958,13 @@ static int createUnionVtab(sqlite3 *db){
0, /* xRelease */
0 /* xRollbackTo */
};
int rc;
return sqlite3_create_module(db, "unionvtab", &unionModule, 0);
rc = sqlite3_create_module(db, "unionvtab", &unionModule, 0);
if( rc==SQLITE_OK ){
rc = sqlite3_create_module(db, "swarmvtab", &unionModule, (void*)db);
}
return rc;
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */

18
manifest
View File

@ -1,5 +1,5 @@
C Avoid\sredundant\scalls\sto\ssqlite3ApiExit()\sin\ssqlite3_step().
D 2017-08-02T19:04:37.188
C Enhance\sthe\scode\sin\sunionvtab.c\sto\salso\sprovide\sthe\s"swarmvtab"\svirtual\stable\nmodule.\sThere\sare\sstill\sseveral\sproblems\son\sthis\sbranch.
D 2017-08-02T19:59:56.307
F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@ -281,7 +281,7 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d
F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
F ext/misc/unionvtab.c 56fd163d2b6d2f4df0078be482fc9a874658ce51cce33f180c08834193449c78
F ext/misc/unionvtab.c f9acc9c3e78e48171bd72735f08948585104b811546cee02d012cbd481432665
F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
@ -1231,6 +1231,7 @@ F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303
F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a
F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
F test/swarmvtab.test 49dc94deb7363a375a980e9d52815533138e3046a4a21d5eb1c312552d408033
F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece
@ -1640,7 +1641,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P bcc6dacb9114df709ef1bde24264c2193d9e39fc7fab024d5ebfc6056033274c
R c2ce69750ff40b8566324e65e78319fb
U drh
Z 4c16d972871bc4f6fb8b1229cdad47da
P 527974d4caba8bce7c89a28ea04a573b14c558657c14d9ad3c64bf1e0884caf8
R 31849be59b061a67f41a14392566e88f
T *branch * union-vtab
T *sym-union-vtab *
T -sym-trunk *
U dan
Z 1bdb6d432069947ff558a9e41aa801b1

2
manifest.uuid
View File

@ -1 +1 @@
527974d4caba8bce7c89a28ea04a573b14c558657c14d9ad3c64bf1e0884caf8
03d94388d62fd0f1fae377d273bbd5561208adc34bd97f7ce27783b30a369fd7

91
test/swarmvtab.test Normal file
View File

@ -0,0 +1,91 @@
# 2017-07-15
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. The
# focus of this file is the "swarmvtab" extension
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix swarmvtab
ifcapable !vtab {
finish_test
return
}
load_static_extension db unionvtab
do_execsql_test 1.0 {
CREATE TABLE t0(a INTEGER PRIMARY KEY, b TEXT);
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<400)
INSERT INTO t0 SELECT i, hex(randomblob(50)) FROM s;
CREATE TABLE dir(f, t, imin, imax);
}
do_test 1.1 {
for {set i 0} {$i < 40} {incr i} {
set iMin [expr $i*10 + 1]
set iMax [expr $iMin+9]
forcedelete "test.db$i"
execsql [subst {
ATTACH 'test.db$i' AS aux;
CREATE TABLE aux.t$i (a INTEGER PRIMARY KEY, b TEXT);
INSERT INTO aux.t$i SELECT * FROM t0 WHERE a BETWEEN $iMin AND $iMax;
DETACH aux;
INSERT INTO dir VALUES('test.db$i', 't$i', $iMin, $iMax);
}]
}
execsql {
CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir');
SELECT count(*) FROM pragma_database_list;
}
} {3}
do_execsql_test 1.2 {
DROP TABLE s1;
SELECT name FROM pragma_database_list;
} {main temp}
do_execsql_test 1.3 {
CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir');
SELECT count(*) FROM s1 WHERE rowid<50;
} {49}
proc do_compare_test {tn where} {
set sql [subst {
SELECT (
SELECT group_concat(a || ',' || b, ',') FROM t0 WHERE $where
)==(
SELECT group_concat(a || ',' || b, ',') FROM s1 WHERE $where
)
}]
uplevel [list do_execsql_test $tn $sql 1]
}
do_compare_test 1.4 "rowid = 55"
do_compare_test 1.5 "rowid BETWEEN 20 AND 100"
do_compare_test 1.6 "rowid > 350"
do_compare_test 1.7 "rowid >= 350"
# The following both each an assert() in SQLite:
#
#do_compare_test 1.8 "rowid >= 200"
#do_test 1.x { db close } {}
#
do_execsql_test 1.x { DROP TABLE s1 }
finish_test