mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enhance the memdb VFS so that it is able to share databases among multiple

Browse Source 
database connections in the same process, as long as the database filename
begins with "/".  This provides a way for threads to share an in-memory
database without the use of shared-cache mode.

FossilOrigin-Name: 533fffc4a39b01c3aba75bd3271fd6ccd9516d9681ed04adbe19bd7de03f4c16
This commit is contained in:
drh 2021-05-12 11:55:59 +00:00
commit 277ae64103
3 changed files with 271 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
manifest
View File

@ -1,5 +1,5 @@
C Fix\sa\sNEVER()\sthat\scan\ssometimes\sbe\strue.\ndbsqlfuzz\s9a86fb3830977e216fde061bfbe1c1e5c2e9bfc4
D 2021-05-11T10:47:41.595
C Enhance\sthe\smemdb\sVFS\sso\sthat\sit\sis\sable\sto\sshare\sdatabases\samong\smultiple\ndatabase\sconnections\sin\sthe\ssame\sprocess,\sas\slong\sas\sthe\sdatabase\sfilename\nbegins\swith\s"/".\s\sThis\sprovides\sa\sway\sfor\sthreads\sto\sshare\san\sin-memory\ndatabase\swithout\sthe\suse\sof\sshared-cache\smode.
D 2021-05-12T11:55:59.991
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -514,7 +514,7 @@ F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
F src/mem2.c b93b8762ab999a29ae7751532dadf0a1ac78040308a5fb1d17fcc365171d67eb
F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6
F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
F src/memdb.c 4dac796ed620f061fc67ddd1060a5ae1e9cdbb6a4e2b03397306864f2af190b3
F src/memdb.c 0c95e0299cc81e19d8695d2e7b81c338a5e77d083aa7849643eafcc2a3b1115a
F src/memjournal.c 431c70a111223a8a6e2e7e9f014afc6c88d818d357d866afc563195f2277d50e
F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8
F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25
@ -1912,7 +1912,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 6df3b03e00b1143be8fed3a39a58ce81063020275aa1ac13d87c84f1ceda6e27
R 888429298506a7a430e45d7ff6f16cee
P f39666e64d6d32420170c54f874d8314eb8c6f91df625f7b28f9ccffb9474dc0 98dae595d861941bb0bcd12126ee02492587c466e6da579a58b5dc4a4d655917
R 1441e09faa39fd11227f415ae28549ff
T +closed 98dae595d861941bb0bcd12126ee02492587c466e6da579a58b5dc4a4d655917
U drh
Z 3810e14a95fb243baca7c6b1006c9619
Z bbd64c76e9984e3f1696dac12550f00b

2
manifest.uuid
View File

@ -1 +1 @@
f39666e64d6d32420170c54f874d8314eb8c6f91df625f7b28f9ccffb9474dc0
533fffc4a39b01c3aba75bd3271fd6ccd9516d9681ed04adbe19bd7de03f4c16

312
src/memdb.c
View File

@ -24,24 +24,81 @@
*/
typedef struct sqlite3_vfs MemVfs;
typedef struct MemFile MemFile;
typedef struct MemStore MemStore;
/* Access to a lower-level VFS that (might) implement dynamic loading,
** access to randomness, etc.
*/
#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
/* An open file */
struct MemFile {
sqlite3_file base; /* IO methods */
/* Storage for a memdb file.
**
** An memdb object can be shared or separate. Shared memdb objects can be
** used by more than one database connection. Mutexes are used by shared
** memdb objects to coordinate access. Separate memdb objects are only
** connected to a single database connection and do not require additional
** mutexes.
**
** Shared memdb objects have .zFName!=0 and .pMutex!=0. They are created
** using "file:/name?vfs=memdb". The first character of the name must be
** "/" or else the object will be a separate memdb object. All shared
** memdb objects are stored in memdb_g.apMemStore[] in an arbitrary order.
**
** Separate memdb objects are created using a name that does not begin
** with "/" or using sqlite3_deserialize().
**
** Access rules for shared MemStore objects:
**
** * .zFName is initialized when the object is created and afterwards
** is unchanged until the object is destroyed. So it can be accessed
** at any time as long as we know the object is not being destroyed,
** which means while either the SQLITE_MUTEX_STATIC_VFS1 or
** .pMutex is held or the object is not part of memdb_g.apMemStore[].
**
** * Can .pMutex can only be changed while holding the
** SQLITE_MUTEX_STATIC_VFS1 mutex or while the object is not part
** of memdb_g.apMemStore[].
**
** * Other fields can only be changed while holding the .pMutex mutex
** or when the .nRef is less than zero and the object is not part of
** memdb_g.apMemStore[].
**
** * The .aData pointer has the added requirement that it can can only
** be changed (for resizing) when nMmap is zero.
**
*/
struct MemStore {
sqlite3_int64 sz; /* Size of the file */
sqlite3_int64 szAlloc; /* Space allocated to aData */
sqlite3_int64 szMax; /* Maximum allowed size of the file */
unsigned char *aData; /* content of the file */
sqlite3_mutex *pMutex; /* Used by shared stores only */
int nMmap; /* Number of memory mapped pages */
unsigned mFlags; /* Flags */
int nRdLock; /* Number of readers */
int nWrLock; /* Number of writers. (Always 0 or 1) */
int nRef; /* Number of users of this MemStore */
char *zFName; /* The filename for shared stores */
};
/* An open file */
struct MemFile {
sqlite3_file base; /* IO methods */
MemStore *pStore; /* The storage */
int eLock; /* Most recent lock against this file */
};
/*
** Global variables for holding the memdb files that are accessible
** to multiple database connections in separate threads.
**
** Must hold SQLITE_MUTEX_STATIC_VFS1 to access any part of this object.
*/
struct MemFS {
int nMemStore; /* Number of shared MemStore objects */
MemStore **apMemStore; /* Array of all shared MemStore objects */
} memdb_g;
/*
** Methods for MemFile
*/
@ -120,19 +177,54 @@ static const sqlite3_io_methods memdb_io_methods = {
memdbUnfetch /* xUnfetch */
};
/*
** Enter/leave the mutex on a MemStore
*/
static void memdbEnter(MemStore *p){
sqlite3_mutex_enter(p->pMutex);
}
static void memdbLeave(MemStore *p){
sqlite3_mutex_leave(p->pMutex);
}
/*
** Close an memdb-file.
**
** The pData pointer is owned by the application, so there is nothing
** to free. Unless the SQLITE_DESERIALIZE_FREEONCLOSE flag is set,
** in which case we own the pData pointer and need to free it.
** Free the underlying MemStore object when its refcount drops to zero
** or less.
*/
static int memdbClose(sqlite3_file *pFile){
MemFile *p = (MemFile *)pFile;
if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ){
sqlite3_free(p->aData);
MemStore *p = ((MemFile*)pFile)->pStore;
memdbEnter(p);
p->nRef--;
if( p->nRef<=0 ){
if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ){
sqlite3_free(p->aData);
}
if( p->zFName ){
int i;
#ifndef SQLITE_MUTEX_OMIT
sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
#endif
sqlite3_mutex_enter(pVfsMutex);
for(i=0; ALWAYS(i<memdb_g.nMemStore); i++){
if( memdb_g.apMemStore[i]==p ){
memdb_g.apMemStore[i] = memdb_g.apMemStore[--memdb_g.nMemStore];
if( memdb_g.nMemStore==0 ){
sqlite3_free(memdb_g.apMemStore);
memdb_g.apMemStore = 0;
}
break;
}
}
sqlite3_mutex_leave(pVfsMutex);
}
memdbLeave(p);
sqlite3_mutex_free(p->pMutex);
sqlite3_free(p);
}else{
memdbLeave(p);
}
return SQLITE_OK;
}
@ -146,20 +238,23 @@ static int memdbRead(
int iAmt,
sqlite_int64 iOfst
){
MemFile *p = (MemFile *)pFile;
MemStore *p = ((MemFile*)pFile)->pStore;
memdbEnter(p);
if( iOfst+iAmt>p->sz ){
memset(zBuf, 0, iAmt);
if( iOfst<p->sz ) memcpy(zBuf, p->aData+iOfst, p->sz - iOfst);
memdbLeave(p);
return SQLITE_IOERR_SHORT_READ;
}
memcpy(zBuf, p->aData+iOfst, iAmt);
memdbLeave(p);
return SQLITE_OK;
}
/*
** Try to enlarge the memory allocation to hold at least sz bytes
*/
static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){
static int memdbEnlarge(MemStore *p, sqlite3_int64 newSz){
unsigned char *pNew;
if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){
return SQLITE_FULL;
@ -185,19 +280,27 @@ static int memdbWrite(
int iAmt,
sqlite_int64 iOfst
){
MemFile *p = (MemFile *)pFile;
if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ) return SQLITE_READONLY;
MemStore *p = ((MemFile*)pFile)->pStore;
memdbEnter(p);
if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ){
/* Can't happen: memdbLock() will return SQLITE_READONLY before
** reaching this point */
memdbLeave(p);
return SQLITE_IOERR_WRITE;
}
if( iOfst+iAmt>p->sz ){
int rc;
if( iOfst+iAmt>p->szAlloc
&& (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK
){
memdbLeave(p);
return rc;
}
if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
p->sz = iOfst+iAmt;
}
memcpy(p->aData+iOfst, z, iAmt);
memdbLeave(p);
return SQLITE_OK;
}
@ -209,10 +312,16 @@ static int memdbWrite(
** the size of a file, never to increase the size.
*/
static int memdbTruncate(sqlite3_file *pFile, sqlite_int64 size){
MemFile *p = (MemFile *)pFile;
if( NEVER(size>p->sz) ) return SQLITE_FULL;
p->sz = size;
return SQLITE_OK;
MemStore *p = ((MemFile*)pFile)->pStore;
int rc = SQLITE_OK;
memdbEnter(p);
if( NEVER(size>p->sz) ){
rc = SQLITE_FULL;
}else{
p->sz = size;
}
memdbLeave(p);
return rc;
}
/*
@ -226,8 +335,10 @@ static int memdbSync(sqlite3_file *pFile, int flags){
** Return the current file-size of an memdb-file.
*/
static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
MemFile *p = (MemFile *)pFile;
MemStore *p = ((MemFile*)pFile)->pStore;
memdbEnter(p);
*pSize = p->sz;
memdbLeave(p);
return SQLITE_OK;
}
@ -235,19 +346,48 @@ static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
** Lock an memdb-file.
*/
static int memdbLock(sqlite3_file *pFile, int eLock){
MemFile *p = (MemFile *)pFile;
if( eLock>SQLITE_LOCK_SHARED
&& (p->mFlags & SQLITE_DESERIALIZE_READONLY)!=0
){
return SQLITE_READONLY;
MemFile *pThis = (MemFile*)pFile;
MemStore *p = pThis->pStore;
int rc = SQLITE_OK;
if( eLock==pThis->eLock ) return SQLITE_OK;
memdbEnter(p);
if( eLock>SQLITE_LOCK_SHARED ){
if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){
rc = SQLITE_READONLY;
}else if( pThis->eLock<=SQLITE_LOCK_SHARED ){
if( p->nWrLock ){
rc = SQLITE_BUSY;
}else{
p->nWrLock = 1;
}
}
}else if( eLock==SQLITE_LOCK_SHARED ){
if( pThis->eLock > SQLITE_LOCK_SHARED ){
assert( p->nWrLock==1 );
p->nWrLock = 0;
}else if( p->nWrLock ){
rc = SQLITE_BUSY;
}else{
p->nRdLock++;
}
}else{
assert( eLock==SQLITE_LOCK_NONE );
if( pThis->eLock>SQLITE_LOCK_SHARED ){
assert( p->nWrLock==1 );
p->nWrLock = 0;
}
assert( p->nRdLock>0 );
p->nRdLock--;
}
p->eLock = eLock;
return SQLITE_OK;
if( rc==SQLITE_OK ) pThis->eLock = eLock;
memdbLeave(p);
return rc;
}
#if 0 /* Never used because memdbAccess() always returns false */
#if 0
/*
** Check if another file-handle holds a RESERVED lock on an memdb-file.
** This interface is only used for crash recovery, which does not
** occur on an in-memory database.
*/
static int memdbCheckReservedLock(sqlite3_file *pFile, int *pResOut){
*pResOut = 0;
@ -255,12 +395,14 @@ static int memdbCheckReservedLock(sqlite3_file *pFile, int *pResOut){
}
#endif
/*
** File control method. For custom operations on an memdb-file.
*/
static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){
MemFile *p = (MemFile *)pFile;
MemStore *p = ((MemFile*)pFile)->pStore;
int rc = SQLITE_NOTFOUND;
memdbEnter(p);
if( op==SQLITE_FCNTL_VFSNAME ){
*(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz);
rc = SQLITE_OK;
@ -278,6 +420,7 @@ static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){
*(sqlite3_int64*)pArg = iLimit;
rc = SQLITE_OK;
}
memdbLeave(p);
return rc;
}
@ -307,20 +450,24 @@ static int memdbFetch(
int iAmt,
void **pp
){
MemFile *p = (MemFile *)pFile;
MemStore *p = ((MemFile*)pFile)->pStore;
memdbEnter(p);
if( iOfst+iAmt>p->sz ){
*pp = 0;
}else{
p->nMmap++;
*pp = (void*)(p->aData + iOfst);
}
memdbLeave(p);
return SQLITE_OK;
}
/* Release a memory-mapped page */
static int memdbUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
MemFile *p = (MemFile *)pFile;
MemStore *p = ((MemFile*)pFile)->pStore;
memdbEnter(p);
p->nMmap--;
memdbLeave(p);
return SQLITE_OK;
}
@ -330,20 +477,79 @@ static int memdbUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
static int memdbOpen(
sqlite3_vfs *pVfs,
const char *zName,
sqlite3_file *pFile,
sqlite3_file *pFd,
int flags,
int *pOutFlags
){
MemFile *p = (MemFile*)pFile;
MemFile *pFile = (MemFile*)pFd;
MemStore *p = 0;
int szName;
if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFd, flags, pOutFlags);
}
memset(p, 0, sizeof(*p));
p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
memset(pFile, 0, sizeof(*p));
szName = sqlite3Strlen30(zName);
if( szName>1 && zName[0]=='/' ){
int i;
#ifndef SQLITE_MUTEX_OMIT
sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
#endif
sqlite3_mutex_enter(pVfsMutex);
for(i=0; i<memdb_g.nMemStore; i++){
if( strcmp(memdb_g.apMemStore[i]->zFName,zName)==0 ){
p = memdb_g.apMemStore[i];
break;
}
}
if( p==0 ){
MemStore **apNew;
p = sqlite3Malloc( sizeof(*p) + szName + 3 );
if( p==0 ){
sqlite3_mutex_leave(pVfsMutex);
return SQLITE_NOMEM;
}
apNew = sqlite3Realloc(memdb_g.apMemStore,
sizeof(apNew[0])*(memdb_g.nMemStore+1) );
if( apNew==0 ){
sqlite3_free(p);
sqlite3_mutex_leave(pVfsMutex);
return SQLITE_NOMEM;
}
apNew[memdb_g.nMemStore++] = p;
memdb_g.apMemStore = apNew;
memset(p, 0, sizeof(*p));
p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE|SQLITE_DESERIALIZE_FREEONCLOSE;
p->szMax = sqlite3GlobalConfig.mxMemdbSize;
p->zFName = (char*)&p[1];
memcpy(p->zFName, zName, szName+1);
p->pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
if( p->pMutex==0 ){
memdb_g.nMemStore--;
sqlite3_free(p);
sqlite3_mutex_leave(pVfsMutex);
return SQLITE_NOMEM;
}
p->nRef = 1;
memdbEnter(p);
}else{
memdbEnter(p);
p->nRef++;
}
sqlite3_mutex_leave(pVfsMutex);
}else{
p = sqlite3Malloc( sizeof(*p) );
if( p==0 ){
return SQLITE_NOMEM;
}
memset(p, 0, sizeof(*p));
p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
p->szMax = sqlite3GlobalConfig.mxMemdbSize;
}
pFile->pStore = p;
assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */
*pOutFlags = flags | SQLITE_OPEN_MEMORY;
pFile->pMethods = &memdb_io_methods;
p->szMax = sqlite3GlobalConfig.mxMemdbSize;
pFd->pMethods = &memdb_io_methods;
memdbLeave(p);
return SQLITE_OK;
}
@ -458,9 +664,14 @@ static int memdbCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
*/
static MemFile *memdbFromDbSchema(sqlite3 *db, const char *zSchema){
MemFile *p = 0;
MemStore *pStore;
int rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
if( rc ) return 0;
if( p->base.pMethods!=&memdb_io_methods ) return 0;
pStore = p->pStore;
memdbEnter(pStore);
if( pStore->zFName!=0 ) p = 0;
memdbLeave(pStore);
return p;
}
@ -496,12 +707,14 @@ unsigned char *sqlite3_serialize(
if( piSize ) *piSize = -1;
if( iDb<0 ) return 0;
if( p ){
if( piSize ) *piSize = p->sz;
MemStore *pStore = p->pStore;
assert( pStore->pMutex==0 );
if( piSize ) *piSize = pStore->sz;
if( mFlags & SQLITE_SERIALIZE_NOCOPY ){
pOut = p->aData;
pOut = pStore->aData;
}else{
pOut = sqlite3_malloc64( p->sz );
if( pOut ) memcpy(pOut, p->aData, p->sz);
pOut = sqlite3_malloc64( pStore->sz );
if( pOut ) memcpy(pOut, pStore->aData, pStore->sz);
}
return pOut;
}
@ -595,15 +808,16 @@ int sqlite3_deserialize(
if( p==0 ){
rc = SQLITE_ERROR;
}else{
p->aData = pData;
MemStore *pStore = p->pStore;
pStore->aData = pData;
pData = 0;
p->sz = szDb;
p->szAlloc = szBuf;
p->szMax = szBuf;
if( p->szMax<sqlite3GlobalConfig.mxMemdbSize ){
p->szMax = sqlite3GlobalConfig.mxMemdbSize;
pStore->sz = szDb;
pStore->szAlloc = szBuf;
pStore->szMax = szBuf;
if( pStore->szMax<sqlite3GlobalConfig.mxMemdbSize ){
pStore->szMax = sqlite3GlobalConfig.mxMemdbSize;
}
p->mFlags = mFlags;
pStore->mFlags = mFlags;
rc = SQLITE_OK;
}