From 8a922f758d198a7df1e3f20b55f3fd657e968d6f Mon Sep 17 00:00:00 2001 From: shaneh Date: Thu, 4 Nov 2010 20:50:27 +0000 Subject: [PATCH] First attempt at a sharding VFS to split large DBs. FossilOrigin-Name: dd4dc8a4269e23ffe0e18438690da6077e17cdad --- manifest | 14 +- manifest.uuid | 2 +- src/tclsqlite.c | 2 + src/test_multiplex.c | 816 +++++++++++++++++++++++++++++++++++++++++++ test/multiplex.test | 306 ++++++++++++++++ 5 files changed, 1133 insertions(+), 7 deletions(-) create mode 100644 src/test_multiplex.c create mode 100644 test/multiplex.test diff --git a/manifest b/manifest index 9c6262987b..7081b63415 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssize\sso\sit's\sreally\s32x32. -D 2010-11-04T20:47:44 +C First\sattempt\sat\sa\ssharding\sVFS\sto\ssplit\slarge\sDBs. +D 2010-11-04T20:50:27 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2c8cefd962eca0147132c7cf9eaa4bb24c656f3f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/sqliteInt.h c63b0340dfdfde18ff255ddccf004edd2d073288 F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44 F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c dc71e2bd0f16dbb9106c1f5e257a64b783574f26 +F src/tclsqlite.c e1c485fa323e3ef02e5b10fe6a016e7638013eb9 F src/test1.c 9e1fe842f72aa41af48ab7b21c3a12975b4c5b37 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 056093cfef69ff4227a6bdb9108564dc7f45e4bc @@ -207,6 +207,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 785edd54f963aefb3c1628124170a56697c68c70 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 09a88f0c111201dc4f8c20470aa1b5f611d59200 +F src/test_multiplex.c be9c1445dca81ab9c77278b85b827cd446c656a8 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec F src/test_osinst.c f408c6a181f2fb04c56273afd5c3e1e82f60392c @@ -558,6 +559,7 @@ F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test c5f4e6a82e04e71820c0f9f64f6733f04c8ae0ae F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 +F test/multiplex.test 257af7decc32a50814adffbb6865e1a7eb9f42c3 F test/mutex1.test 5b71777fc127509cd257910c8db799de557a02de F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test a44e04df1486fcfb02d32468cbcd3c8e1e433723 @@ -883,7 +885,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P cc9d9a12c3b19320b0ecfa78a6cee3d668004988 -R 0b84c10fec00628f99d964887ecf071d +P 62c494f6947e53808404081c8f48cc327ef814e7 +R 081ff33d1b2f0f16499724ab46c0e8b9 U shaneh -Z 2c70e7d440c064ec216e31a9c4e6aaa8 +Z fa2a353cd6df65e68d308d8dfcf83710 diff --git a/manifest.uuid b/manifest.uuid index 193df92131..6328c230be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62c494f6947e53808404081c8f48cc327ef814e7 \ No newline at end of file +dd4dc8a4269e23ffe0e18438690da6077e17cdad \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 471b0d3fb1..13796bed2f 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3579,6 +3579,7 @@ static void init_all(Tcl_Interp *interp){ extern int SqlitetestStat_Init(Tcl_Interp*); extern int Sqlitetestrtree_Init(Tcl_Interp*); extern int Sqlitequota_Init(Tcl_Interp*); + extern int Sqlitemultiplex_Init(Tcl_Interp*); Sqliteconfig_Init(interp); Sqlitetest1_Init(interp); @@ -3609,6 +3610,7 @@ static void init_all(Tcl_Interp *interp){ SqlitetestStat_Init(interp); Sqlitetestrtree_Init(interp); Sqlitequota_Init(interp); + Sqlitemultiplex_Init(interp); Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); diff --git a/src/test_multiplex.c b/src/test_multiplex.c new file mode 100644 index 0000000000..d8c047300b --- /dev/null +++ b/src/test_multiplex.c @@ -0,0 +1,816 @@ +/* +** 2010 October 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a VFS "shim" - a layer that sits in between the +** pager and the real VFS. +** +** This particular shim enforces a multiplex system on DB files. +** This shim shards/partitions a single DB file into smaller +** "chunks" such that the total DB file size may exceed the maximum +** file size of the underlying file system. +** +*/ +#include "sqlite3.h" +#include +#include +#include "sqliteInt.h" + +/************************ Shim Definitions ******************************/ + +#define SQLITE_MULTIPLEX_CHUNK_SIZE 0x80000000 +#define SQLITE_MULTIPLEX_MAX_CHUNKS 32 + +/************************ Object Definitions ******************************/ + +/* Forward declaration of all object types */ +typedef struct multiplexGroup multiplexGroup; +typedef struct multiplexConn multiplexConn; + +/* +** A "multiplex group" is a collection of files that collectively +** makeup a single SQLite DB file. This allows the size of the DB +** to exceed the limits imposed by the file system. +** +** There is an instance of the following object for each defined multiplex +** group. +*/ +struct multiplexGroup { + sqlite3_file *pReal[SQLITE_MULTIPLEX_MAX_CHUNKS]; /* Handles to each chunk */ + char bOpen[SQLITE_MULTIPLEX_MAX_CHUNKS]; /* 0 if chunk not opened */ + char *zName; /* Base filename of this group */ + int nName; /* Length of base filename */ + int flags; /* Flags used for original opening */ + multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ +}; + +/* +** An instance of the following object represents each open connection +** to a file that is multiplex'ed. This object is a +** subclass of sqlite3_file. The sqlite3_file object for the underlying +** VFS is appended to this structure. +*/ +struct multiplexConn { + sqlite3_file base; /* Base class - must be first */ + multiplexGroup *pGroup; /* The underlying group of files */ +}; + +/************************* Global Variables **********************************/ +/* +** All global variables used by this file are containing within the following +** gMultiplex structure. +*/ +static struct { + /* The pOrigVfs is the real, original underlying VFS implementation. + ** Most operations pass-through to the real VFS. This value is read-only + ** during operation. It is only modified at start-time and thus does not + ** require a mutex. + */ + sqlite3_vfs *pOrigVfs; + + /* The sThisVfs is the VFS structure used by this shim. It is initialized + ** at start-time and thus does not require a mutex + */ + sqlite3_vfs sThisVfs; + + /* The sIoMethods defines the methods used by sqlite3_file objects + ** associated with this shim. It is initialized at start-time and does + ** not require a mutex. + ** + ** When the underlying VFS is called to open a file, it might return + ** either a version 1 or a version 2 sqlite3_file object. This shim + ** has to create a wrapper sqlite3_file of the same version. Hence + ** there are two I/O method structures, one for version 1 and the other + ** for version 2. + */ + sqlite3_io_methods sIoMethodsV1; + sqlite3_io_methods sIoMethodsV2; + + /* True when this shim as been initialized. + */ + int isInitialized; + + /* For run-time access any of the other global data structures in this + ** shim, the following mutex must be held. + */ + sqlite3_mutex *pMutex; + + /* List of multiplexGroup objects. + */ + multiplexGroup *pGroups; + + /* Chunk params + */ + int nChunkSize; + int nMaxChunks; + +} gMultiplex; + +/************************* Utility Routines *********************************/ +/* +** Acquire and release the mutex used to serialize access to the +** list of multiplexGroups. +*/ +static void multiplexEnter(void){ sqlite3_mutex_enter(gMultiplex.pMutex); } +static void multiplexLeave(void){ sqlite3_mutex_leave(gMultiplex.pMutex); } + +/* Translate an sqlite3_file* that is really a multiplexGroup* into +** the sqlite3_file* for the underlying original VFS. +*/ +static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){ + multiplexGroup *pGroup = pConn->pGroup; + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ + if( iChunkpReal[iChunk]; /* Real file descriptor */ + if( !pGroup->bOpen[iChunk] ){ + pGroup->zName[pGroup->nName] = '\0'; + if( iChunk ) sqlite3_snprintf(pGroup->nName+6, pGroup->zName+pGroup->nName, "-%04d", iChunk); + *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->zName, pSubOpen, pGroup->flags, pOutFlags); + if( *rc==SQLITE_OK ){ + pGroup->bOpen[iChunk] = -1; + return pSubOpen; + } + return NULL; + } + *rc = SQLITE_OK; + return pSubOpen; + } + *rc = SQLITE_ERROR; + return NULL; +} + +/************************* VFS Method Wrappers *****************************/ +/* +** This is the xOpen method used for the "multiplex" VFS. +** +** Most of the work is done by the underlying original VFS. This method +** simply links the new file into the appropriate multiplex group if it is a +** file that needs to be tracked. +*/ +static int multiplexOpen( + sqlite3_vfs *pVfs, /* The multiplex VFS */ + const char *zName, /* Name of file to be opened */ + sqlite3_file *pConn, /* Fill in this file descriptor */ + int flags, /* Flags to control the opening */ + int *pOutFlags /* Flags showing results of opening */ +){ + int rc; /* Result code */ + multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ + multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ + sqlite3_file *pSubOpen; /* Real file descriptor */ + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ + int nName = sqlite3Strlen30(zName); + int i; + + UNUSED_PARAMETER(pVfs); + + /* If the file is not a main database file or a WAL, then use the + ** normal xOpen method. + */ + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))==0 ){ + return pOrigVfs->xOpen(pOrigVfs, zName, pConn, flags, pOutFlags); + } + + /* We need to create a group structure and manage + ** access to this group of files. + */ + multiplexEnter(); + pMultiplexOpen = (multiplexConn*)pConn; + /* -0000\0 */ + pGroup = sqlite3_malloc( sizeof(multiplexGroup) + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks) + nName + 6 ); + if( pGroup==0 ){ + rc=SQLITE_NOMEM; + }else{ + pMultiplexOpen->pGroup = pGroup; + memset(pGroup, 0, sizeof(multiplexGroup) + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks) + nName + 6); + for(i=0; ipReal[i] = (sqlite3_file *)((char *)&pGroup[1] + (pOrigVfs->szOsFile*i)); + } + pGroup->zName = (char *)&pGroup[1] + (pOrigVfs->szOsFile*gMultiplex.nMaxChunks); + memcpy(pGroup->zName, zName, nName+1); + pGroup->nName = nName; + pGroup->flags = flags; + pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags); + if( pSubOpen ){ + if( pSubOpen->pMethods->iVersion==1 ){ + pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1; + }else{ + pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV2; + } + /* place this group at the head of our list */ + pGroup->pNext = gMultiplex.pGroups; + if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup; + gMultiplex.pGroups = pGroup; + }else{ + sqlite3_free(pGroup); + } + } + multiplexLeave(); + return rc; +} + +/************************ I/O Method Wrappers *******************************/ + +/* xClose requests get passed through to the original VFS. +** We loop over all open chunk handles and close them. +** The group structure for this file is unlinked from +** our list of groups and freed. +*/ +static int multiplexClose(sqlite3_file *pConn){ + multiplexConn *p = (multiplexConn*)pConn; + multiplexGroup *pGroup = p->pGroup; + int rc = SQLITE_OK; + int i; + multiplexEnter(); + for(i=0; ibOpen[i] ){ + sqlite3_file *pSubOpen = pGroup->pReal[i]; + int rc2 = pSubOpen->pMethods->xClose(pSubOpen); + if( rc2!=SQLITE_OK ) rc = rc2; + pGroup->bOpen[i] = 0; + } + } + if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev; + if( pGroup->pPrev ){ + pGroup->pPrev->pNext = pGroup->pNext; + }else{ + gMultiplex.pGroups = pGroup->pNext; + } + sqlite3_free(pGroup); + multiplexLeave(); + return rc; +} + +/* Pass xRead requests thru to the original VFS after +** determining the correct chunk to operate on. +*/ +static int multiplexRead( + sqlite3_file *pConn, + void *pBuf, + int iAmt, + sqlite3_int64 iOfst +){ + multiplexConn *p = (multiplexConn*)pConn; + int rc = SQLITE_OK; + multiplexEnter(); + while( iAmt > 0 ){ + int i = (int)(iOfst/gMultiplex.nChunkSize); + sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + if( pSubOpen ){ + int extra = ((int)(iOfst % gMultiplex.nChunkSize) + iAmt) - gMultiplex.nChunkSize; + if( extra<0 ) extra = 0; + iAmt -= extra; + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst%gMultiplex.nChunkSize); + if( rc!=SQLITE_OK ) break; + pBuf = (char *)pBuf + iAmt; + iOfst += iAmt; + iAmt = extra; + }else{ + rc = SQLITE_IOERR_READ; + break; + } + } + multiplexLeave(); + return rc; +} + +/* Pass xWrite requests thru to the original VFS after +** determining the correct chunk to operate on. +*/ +static int multiplexWrite( + sqlite3_file *pConn, + const void *pBuf, + int iAmt, + sqlite3_int64 iOfst +){ + multiplexConn *p = (multiplexConn*)pConn; + int rc = SQLITE_OK; + multiplexEnter(); + while( iAmt > 0 ){ + int i = (int)(iOfst/gMultiplex.nChunkSize); + sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + if( pSubOpen ){ + int extra = ((int)(iOfst % gMultiplex.nChunkSize) + iAmt) - gMultiplex.nChunkSize; + if( extra<0 ) extra = 0; + iAmt -= extra; + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst%gMultiplex.nChunkSize); + if( rc!=SQLITE_OK ) break; + pBuf = (char *)pBuf + iAmt; + iOfst += iAmt; + iAmt = extra; + }else{ + rc = SQLITE_IOERR_WRITE; + break; + } + } + multiplexLeave(); + return rc; +} + +/* Pass xTruncate requests thru to the original VFS after +** determining the correct chunk to operate on. Delete any +** chunks above the truncate mark. +*/ +static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ + multiplexConn *p = (multiplexConn*)pConn; + multiplexGroup *pGroup = p->pGroup; + int rc = SQLITE_OK; + int rc2; + int i; + sqlite3_file *pSubOpen; + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ + multiplexEnter(); + /* delete the chunks above the truncate limit */ + for(i=(int)(size/gMultiplex.nChunkSize)+1; ibOpen[i] ){ + pSubOpen = pGroup->pReal[i]; + rc2 = pSubOpen->pMethods->xClose(pSubOpen); + if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; + } + pGroup->zName[pGroup->nName] = '\0'; + if( i ) sqlite3_snprintf(pGroup->nName+6, pGroup->zName+pGroup->nName, "-%04d", i); + rc2 = pOrigVfs->xDelete(pOrigVfs, pGroup->zName, 0); + if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; + } + pSubOpen = multiplexSubOpen(p, (int)(size/gMultiplex.nChunkSize), &rc2, NULL); + if( pSubOpen ){ + rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size%gMultiplex.nChunkSize); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + rc = SQLITE_IOERR_TRUNCATE; + } + multiplexLeave(); + return rc; +} + +/* Pass xSync requests through to the original VFS without change +*/ +static int multiplexSync(sqlite3_file *pConn, int flags){ + multiplexConn *p = (multiplexConn*)pConn; + multiplexGroup *pGroup = p->pGroup; + int rc = SQLITE_OK; + int i; + multiplexEnter(); + for(i=0; ibOpen[i] ){ + sqlite3_file *pSubOpen = pGroup->pReal[i]; + int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags); + if( rc2!=SQLITE_OK ) rc = rc2; + } + } + multiplexLeave(); + return rc; +} + +/* Pass xFileSize requests through to the original VFS. +** Aggregate the size of all the chunks before returning. +*/ +static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ + multiplexConn *p = (multiplexConn*)pConn; + multiplexGroup *pGroup = p->pGroup; + int rc = SQLITE_OK; + int rc2; + int i; + multiplexEnter(); + *pSize = 0; + for(i=0; ibOpen[i] ){ + pSubOpen = pGroup->pReal[i]; + }else{ + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ + int exists = 0; + pGroup->zName[pGroup->nName] = '\0'; + if( i ) sqlite3_snprintf(pGroup->nName+6, pGroup->zName+pGroup->nName, "-%04d", i); + rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->zName, SQLITE_ACCESS_EXISTS, &exists); + if( rc2==SQLITE_OK && exists){ + /* if it exists, open it */ + pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + }else{ + /* stop at first "gap" */ + break; + } + } + if( pSubOpen ){ + rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); + if( rc2!=SQLITE_OK ){ + rc = rc2; + }else{ + *pSize += sz; + } + }else{ + break; + } + } + multiplexLeave(); + return rc; +} + +/* Pass xLock requests through to the original VFS unchanged. +*/ +static int multiplexLock(sqlite3_file *pConn, int lock){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xLock(pSubOpen, lock); + } + return SQLITE_BUSY; +} + +/* Pass xUnlock requests through to the original VFS unchanged. +*/ +static int multiplexUnlock(sqlite3_file *pConn, int lock){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xUnlock(pSubOpen, lock); + } + return SQLITE_IOERR_UNLOCK; +} + +/* Pass xCheckReservedLock requests through to the original VFS unchanged. +*/ +static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); + } + return SQLITE_IOERR_CHECKRESERVEDLOCK; +} + +/* Pass xFileControl requests through to the original VFS unchanged. +*/ +static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen; + if ( op==SQLITE_FCNTL_SIZE_HINT || op==SQLITE_FCNTL_CHUNK_SIZE ) return SQLITE_OK; + pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); + } + return SQLITE_ERROR; +} + +/* Pass xSectorSize requests through to the original VFS unchanged. +*/ +static int multiplexSectorSize(sqlite3_file *pConn){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xSectorSize(pSubOpen); + } + return SQLITE_DEFAULT_SECTOR_SIZE; +} + +/* Pass xDeviceCharacteristics requests through to the original VFS unchanged. +*/ +static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); + } + return 0; +} + +/* Pass xShmMap requests through to the original VFS unchanged. +*/ +static int multiplexShmMap( + sqlite3_file *pConn, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend, pp); + } + return SQLITE_IOERR; +} + +/* Pass xShmLock requests through to the original VFS unchanged. +*/ +static int multiplexShmLock( + sqlite3_file *pConn, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); + } + return SQLITE_BUSY; +} + +/* Pass xShmBarrier requests through to the original VFS unchanged. +*/ +static void multiplexShmBarrier(sqlite3_file *pConn){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + pSubOpen->pMethods->xShmBarrier(pSubOpen); + } +} + +/* Pass xShmUnmap requests through to the original VFS unchanged. +*/ +static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ + multiplexConn *p = (multiplexConn*)pConn; + int rc; + sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + if( pSubOpen ){ + return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); + } + return SQLITE_OK; +} + +/************************** Public Interfaces *****************************/ +/* +** Initialize the multiplex VFS shim. Use the VFS named zOrigVfsName +** as the VFS that does the actual work. Use the default if +** zOrigVfsName==NULL. +** +** The multiplex VFS shim is named "multiplex". It will become the default +** VFS if makeDefault is non-zero. +** +** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once +** during start-up. +*/ +int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ + sqlite3_vfs *pOrigVfs; + if( gMultiplex.isInitialized ) return SQLITE_MISUSE; + pOrigVfs = sqlite3_vfs_find(zOrigVfsName); + if( pOrigVfs==0 ) return SQLITE_ERROR; + assert( pOrigVfs!=&gMultiplex.sThisVfs ); + gMultiplex.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( !gMultiplex.pMutex ){ + return SQLITE_NOMEM; + } + gMultiplex.nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; + gMultiplex.nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS; + gMultiplex.pGroups = NULL; + gMultiplex.isInitialized = 1; + gMultiplex.pOrigVfs = pOrigVfs; + gMultiplex.sThisVfs = *pOrigVfs; + gMultiplex.sThisVfs.szOsFile += sizeof(multiplexConn); + gMultiplex.sThisVfs.zName = "multiplex"; + gMultiplex.sThisVfs.xOpen = multiplexOpen; + gMultiplex.sIoMethodsV1.iVersion = 1; + gMultiplex.sIoMethodsV1.xClose = multiplexClose; + gMultiplex.sIoMethodsV1.xRead = multiplexRead; + gMultiplex.sIoMethodsV1.xWrite = multiplexWrite; + gMultiplex.sIoMethodsV1.xTruncate = multiplexTruncate; + gMultiplex.sIoMethodsV1.xSync = multiplexSync; + gMultiplex.sIoMethodsV1.xFileSize = multiplexFileSize; + gMultiplex.sIoMethodsV1.xLock = multiplexLock; + gMultiplex.sIoMethodsV1.xUnlock = multiplexUnlock; + gMultiplex.sIoMethodsV1.xCheckReservedLock = multiplexCheckReservedLock; + gMultiplex.sIoMethodsV1.xFileControl = multiplexFileControl; + gMultiplex.sIoMethodsV1.xSectorSize = multiplexSectorSize; + gMultiplex.sIoMethodsV1.xDeviceCharacteristics = multiplexDeviceCharacteristics; + gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1; + gMultiplex.sIoMethodsV2.iVersion = 2; + gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap; + gMultiplex.sIoMethodsV2.xShmLock = multiplexShmLock; + gMultiplex.sIoMethodsV2.xShmBarrier = multiplexShmBarrier; + gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap; + sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault); + return SQLITE_OK; +} + +/* +** Shutdown the multiplex system. +** +** All SQLite database connections must be closed before calling this +** routine. +** +** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while +** shutting down in order to free all remaining multiplex groups. +*/ +int sqlite3_multiplex_shutdown(void){ + if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE; + if( gMultiplex.pGroups ) return SQLITE_MISUSE; + gMultiplex.isInitialized = 0; + sqlite3_mutex_free(gMultiplex.pMutex); + sqlite3_vfs_unregister(&gMultiplex.sThisVfs); + memset(&gMultiplex, 0, sizeof(gMultiplex)); + return SQLITE_OK; +} + +/* +** Adjust chunking params. VFS should be initialized first. +** No files should be open. Re-intializing will reset these +** to the default. +*/ +int sqlite3_multiplex_set( + int nChunkSize, /* Max chunk size */ + int nMaxChunks /* Max number of chunks */ +){ + if( !gMultiplex.isInitialized ) return SQLITE_MISUSE; + if( gMultiplex.pGroups ) return SQLITE_MISUSE; + if( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS ) return SQLITE_MISUSE; + multiplexEnter(); + gMultiplex.nChunkSize = nChunkSize; + gMultiplex.nMaxChunks = nMaxChunks; + multiplexLeave(); + return SQLITE_OK; +} + +/***************************** Test Code ***********************************/ +#ifdef SQLITE_TEST +#include + +extern const char *sqlite3TestErrorName(int); + + +/* +** tclcmd: sqlite3_multiplex_initialize NAME MAKEDEFAULT +*/ +static int test_multiplex_initialize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zName; /* Name of new multiplex VFS */ + int makeDefault; /* True to make the new VFS the default */ + int rc; /* Value returned by multiplex_initialize() */ + + UNUSED_PARAMETER(clientData); + + /* Process arguments */ + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME MAKEDEFAULT"); + return TCL_ERROR; + } + zName = Tcl_GetString(objv[1]); + if( Tcl_GetBooleanFromObj(interp, objv[2], &makeDefault) ) return TCL_ERROR; + if( zName[0]=='\0' ) zName = 0; + + /* Call sqlite3_multiplex_initialize() */ + rc = sqlite3_multiplex_initialize(zName, makeDefault); + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + + return TCL_OK; +} + +/* +** tclcmd: sqlite3_multiplex_shutdown +*/ +static int test_multiplex_shutdown( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; /* Value returned by multiplex_shutdown() */ + + UNUSED_PARAMETER(clientData); + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + /* Call sqlite3_multiplex_shutdown() */ + rc = sqlite3_multiplex_shutdown(); + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + + return TCL_OK; +} + +/* +** tclcmd: sqlite3_multiplex_set CHUNK_SIZE MAX_CHUNKS +*/ +static int test_multiplex_set( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int nChunkSize; /* Max chunk size */ + int nMaxChunks; /* Max number of chunks */ + int rc; /* Value returned by sqlite3_multiplex_set() */ + + UNUSED_PARAMETER(clientData); + + /* Process arguments */ + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHUNK_SIZE MAX_CHUNKS"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &nChunkSize) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &nMaxChunks) ) return TCL_ERROR; + + if( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS ){ + Tcl_WrongNumArgs(interp, 1, objv, "MAX_CHUNKS > SQLITE_MULTIPLEX_MAX_CHUNKS"); + return TCL_ERROR; + } + + /* Invoke sqlite3_multiplex_set() */ + rc = sqlite3_multiplex_set(nChunkSize, nMaxChunks); + + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + return TCL_OK; +} + +/* +** tclcmd: sqlite3_multiplex_dump +*/ +static int test_multiplex_dump( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + Tcl_Obj *pResult; + Tcl_Obj *pGroupTerm; + multiplexGroup *pGroup; + int i; + int nChunks = 0; + + UNUSED_PARAMETER(clientData); + UNUSED_PARAMETER(objc); + UNUSED_PARAMETER(objv); + + pResult = Tcl_NewObj(); + multiplexEnter(); + for(pGroup=gMultiplex.pGroups; pGroup; pGroup=pGroup->pNext){ + pGroupTerm = Tcl_NewObj(); + + pGroup->zName[pGroup->nName] = '\0'; + Tcl_ListObjAppendElement(interp, pGroupTerm, + Tcl_NewStringObj(pGroup->zName, -1)); + Tcl_ListObjAppendElement(interp, pGroupTerm, + Tcl_NewIntObj(pGroup->nName)); + Tcl_ListObjAppendElement(interp, pGroupTerm, + Tcl_NewIntObj(pGroup->flags)); + + /* count number of chunks with open handles */ + for(i=0; ibOpen[i] ) nChunks++; + } + Tcl_ListObjAppendElement(interp, pGroupTerm, + Tcl_NewIntObj(nChunks)); + + Tcl_ListObjAppendElement(interp, pGroupTerm, + Tcl_NewIntObj(gMultiplex.nChunkSize)); + Tcl_ListObjAppendElement(interp, pGroupTerm, + Tcl_NewIntObj(gMultiplex.nMaxChunks)); + + Tcl_ListObjAppendElement(interp, pResult, pGroupTerm); + } + multiplexLeave(); + Tcl_SetObjResult(interp, pResult); + return TCL_OK; +} + +/* +** This routine registers the custom TCL commands defined in this +** module. This should be the only procedure visible from outside +** of this module. +*/ +int Sqlitemultiplex_Init(Tcl_Interp *interp){ + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + } aCmd[] = { + { "sqlite3_multiplex_initialize", test_multiplex_initialize }, + { "sqlite3_multiplex_shutdown", test_multiplex_shutdown }, + { "sqlite3_multiplex_set", test_multiplex_set }, + { "sqlite3_multiplex_dump", test_multiplex_dump }, + }; + int i; + + for(i=0; i