From 2491de28f129fca253040834755b26985cbdb775 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Feb 2016 20:14:55 +0000 Subject: [PATCH 1/5] Extend the code in memjournal.c so that it subsumes the role of journal.c. And (untested) can flush journal or statement journal files to disk after they grow to a specified size. FossilOrigin-Name: e0b0b4316531fc8afa74b4882d9c74a91030ec69 --- manifest | 25 ++-- manifest.uuid | 2 +- src/journal.c | 2 + src/memjournal.c | 362 ++++++++++++++++++++++++++++++++++------------- src/os.c | 2 +- src/pager.c | 24 +--- src/sqliteInt.h | 5 +- 7 files changed, 288 insertions(+), 134 deletions(-) diff --git a/manifest b/manifest index 706e06deb0..7546ccd6d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tighter\sdescription\sof\sI/O\sstats\sin\sthe\sshell.\s\sShow\sI/O\sstats\son\sspeedtest1.c. -D 2016-02-27T19:19:22.849 +C Extend\sthe\scode\sin\smemjournal.c\sso\sthat\sit\ssubsumes\sthe\srole\sof\sjournal.c.\sAnd\s(untested)\scan\sflush\sjournal\sor\sstatement\sjournal\sfiles\sto\sdisk\safter\sthey\sgrow\sto\sa\sspecified\ssize. +D 2016-02-27T20:14:55.339 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 4f319afb7c049d40aff7af6e8c4e7cc2ba18e079 @@ -311,7 +311,7 @@ F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 9ca97272e9f74ed0efddf3b4350ee12740cebbef -F src/journal.c fe3a3e2559ce3ce9d371afd30fbabbc074174575 +F src/journal.c 673cbdde5676eb0c55848f561575d45b609c820d F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 9e2a41adcaff16ebc1ebff1f336cbf33de55396f F src/main.c be9309f442ec291177642d2e48e82290e0951f4b @@ -321,7 +321,7 @@ F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memjournal.c 349a04fb803db40532cde0993e39758f1acaecce +F src/memjournal.c 8d045e50120e9988e63bb0af69187e1e5e35122f F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 @@ -329,14 +329,14 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c ca9a104b677328ee037cfdf1a54a16fd1805e8c9 +F src/os.c f89e3ca1c2e3d5015b847aec60371c474acbac82 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 3a6f20736dfb8a0949cdd66553fdf59f6604be35 F src/os_win.c f0d7aa603eb6262143d7169a222aea07c4fca91d F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c d034c69b958c01289eb8070cbf902e1a68cd7e0b +F src/pager.c e2bf43b506447d649e9d9266e988007ae7248699 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y c3ce2c4a7cbf0b699239be6b2a945c5cb51875e2 F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df @@ -354,7 +354,7 @@ F src/shell.c 5e0ab1e708dc294330ccd8230536e1801f60822e F src/sqlite.h.in 57d2a02b14c9ec4f7cb294153eaf62294dc5aa68 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h 63c0e1b5b5d608a1ba2303fe3a554b859a120406 +F src/sqliteInt.h 3c4ed7e5dcb8b7d4cd4ab258ee7d0e0d0368f014 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -1451,7 +1451,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c36948f16b58fe8042c37d0df634308b4e48217 -R 2b473c8c6b36608ca875feba0d8cbb83 -U drh -Z 565e8454d2408a569a86791d9e6b12a6 +P f681d800340e0e710f73d0f7c09101f899249183 +R f456c30e495865d50fc2c9c2ddb305ed +T *branch * memjournal-exp +T *sym-memjournal-exp * +T -sym-trunk * +U dan +Z dad681f00ca4c98b68e43d3727f3c92e diff --git a/manifest.uuid b/manifest.uuid index 546a6418d6..b15150144a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f681d800340e0e710f73d0f7c09101f899249183 \ No newline at end of file +e0b0b4316531fc8afa74b4882d9c74a91030ec69 \ No newline at end of file diff --git a/src/journal.c b/src/journal.c index a5cf8c8e2f..da59db0e50 100644 --- a/src/journal.c +++ b/src/journal.c @@ -24,6 +24,7 @@ ** buffer, or ** 2) The sqlite3JournalCreate() function is called. */ +#if 0 #ifdef SQLITE_ENABLE_ATOMIC_WRITE #include "sqliteInt.h" @@ -254,3 +255,4 @@ int sqlite3JournalSize(sqlite3_vfs *pVfs){ return (pVfs->szOsFile+sizeof(JournalFile)); } #endif +#endif diff --git a/src/memjournal.c b/src/memjournal.c index 62594530e0..6e1bf4edf7 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -21,24 +21,28 @@ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; -/* Space to hold the rollback journal is allocated in increments of -** this many bytes. -** -** The size chosen is a little less than a power of two. That way, -** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This minimizes wasted space in power-of-two -** memory allocators. -*/ -#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) - /* ** The rollback journal is composed of a linked list of these structures. +** +** The zChunk array is always at least 8 bytes in size - usually much more. +** Its actual size is stored in the MemJournal.nChunkSize variable. */ struct FileChunk { FileChunk *pNext; /* Next chunk in the journal */ - u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ + u8 zChunk[8]; /* Content of this chunk */ }; +/* +** By default, allocate this many bytes of memory for each FileChunk object. +*/ +#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 + +/* +** For chunk size nChunkSize, return the number of bytes that should +** be allocated for each FileChunk structure. +*/ +#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) + /* ** An instance of this object serves as a cursor into the rollback journal. ** The cursor can be either for reading or writing. @@ -49,14 +53,23 @@ struct FilePoint { }; /* -** This subclass is a subclass of sqlite3_file. Each open memory-journal +** This structure is a subclass of sqlite3_file. Each open memory-journal ** is an instance of this class. */ struct MemJournal { - sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + int nChunkSize; /* In-memory chunk-size */ + + int nBuf; /* Bytes of data before flushing */ + int nSize; /* Bytes of data currently in memory */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ + + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + const char *zJournal; /* Name of the journal file */ + sqlite3_file *pReal; /* The "real" underlying file descriptor */ }; /* @@ -70,41 +83,96 @@ static int memjrnlRead( sqlite_int64 iOfst /* Begin reading at this offset */ ){ MemJournal *p = (MemJournal *)pJfd; - u8 *zOut = zBuf; - int nRead = iAmt; - int iChunkOffset; - FileChunk *pChunk; - - /* SQLite never tries to read past the end of a rollback journal file */ - assert( iOfst+iAmt<=p->endpoint.iOffset ); - - if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ - sqlite3_int64 iOff = 0; - for(pChunk=p->pFirst; - ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; - pChunk=pChunk->pNext - ){ - iOff += JOURNAL_CHUNKSIZE; - } + if( p->pReal ){ + return sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); + }else if( (iAmt+iOfst)>p->endpoint.iOffset ){ + return SQLITE_IOERR_SHORT_READ; }else{ - pChunk = p->readpoint.pChunk; - } + u8 *zOut = zBuf; + int nRead = iAmt; + int iChunkOffset; + FileChunk *pChunk; - iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); - do { - int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; - int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); - memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); - zOut += nCopy; - nRead -= iSpace; - iChunkOffset = 0; - } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = iOfst+iAmt; - p->readpoint.pChunk = pChunk; + if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ + sqlite3_int64 iOff = 0; + for(pChunk=p->pFirst; + ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; + pChunk=pChunk->pNext + ){ + iOff += p->nChunkSize; + } + }else{ + pChunk = p->readpoint.pChunk; + } + + iChunkOffset = (int)(iOfst%p->nChunkSize); + do { + int iSpace = p->nChunkSize - iChunkOffset; + int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); + memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + zOut += nCopy; + nRead -= iSpace; + iChunkOffset = 0; + } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); + p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.pChunk = pChunk; + } return SQLITE_OK; } +/* +** Free the list of FileChunk structures headed at MemJournal.pFirst. +*/ +static void memjrnlFreeChunks(MemJournal *p){ + FileChunk *pIter; + FileChunk *pNext; + for(pIter=p->pFirst; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } + p->pFirst = 0; +} + +/* +** Flush the contents of memory to a real file on disk. +*/ +static int createFile(MemJournal *p){ + int rc = SQLITE_OK; + if( !p->pReal ){ + sqlite3_file *pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); + if( rc==SQLITE_OK ){ + int nChunk = p->nChunkSize; + i64 iOff = 0; + FileChunk *pIter; + p->pReal = pReal; + for(pIter=p->pFirst; pIter && rc==SQLITE_OK; pIter=pIter->pNext){ + int nWrite = nChunk; + if( pIter==p->endpoint.pChunk ){ + nWrite = p->endpoint.iOffset % p->nChunkSize; + if( nWrite==0 ) nWrite = p->nChunkSize; + } + rc = sqlite3OsWrite(pReal, pIter->zChunk, nWrite, iOff); + iOff += nWrite; + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while writing to the file, close it before + ** returning. This way, SQLite uses the in-memory journal data to + ** roll back changes made to the internal page-cache before this + ** function was called. */ + sqlite3OsClose(pReal); + p->pReal = 0; + }else{ + /* No error has occurred. Free the in-memory buffers. */ + memjrnlFreeChunks(p); + } + } + } + return rc; +} + + /* ** Write data to the file. */ @@ -118,38 +186,61 @@ static int memjrnlWrite( int nWrite = iAmt; u8 *zWrite = (u8 *)zBuf; - /* An in-memory journal file should only ever be appended to. Random - ** access writes are not required by sqlite. - */ - assert( iOfst==p->endpoint.iOffset ); - UNUSED_PARAMETER(iOfst); + /* If the file has already been created on disk. */ + if( p->pReal ){ + return sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); + } - while( nWrite>0 ){ - FileChunk *pChunk = p->endpoint.pChunk; - int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); - int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); - - if( iChunkOffset==0 ){ - /* New chunk is required to extend the file. */ - FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); - if( !pNew ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - pNew->pNext = 0; - if( pChunk ){ - assert( p->pFirst ); - pChunk->pNext = pNew; - }else{ - assert( !p->pFirst ); - p->pFirst = pNew; - } - p->endpoint.pChunk = pNew; + /* If the file should be created now. */ + else if( p->nBuf>0 && (iAmt+iOfst)>p->nBuf ){ + int rc = createFile(p); + if( rc==SQLITE_OK ){ + rc = memjrnlWrite(pJfd, zBuf, iAmt, iOfst); } + return rc; + } - memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); - zWrite += iSpace; - nWrite -= iSpace; - p->endpoint.iOffset += iSpace; + /* If the contents of this write should be stored in memory */ + else{ + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required. The only exception to this is when + ** the in-memory journal is being used by a connection using the + ** atomic-write optimization. In this case the first 28 bytes of the + ** journal file may be written as part of committing the transaction. */ + assert( iOfst==p->endpoint.iOffset || iOfst==0 ); + if( iOfst==0 && p->pFirst ){ + assert( p->nChunkSize>iAmt ); + memcpy(p->pFirst->zChunk, zBuf, iAmt); + }else{ + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); + int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + p->nSize = iAmt + iOfst; + } } return SQLITE_OK; @@ -160,16 +251,16 @@ static int memjrnlWrite( */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; - FileChunk *pChunk; - assert(size==0); - UNUSED_PARAMETER(size); - pChunk = p->pFirst; - while( pChunk ){ - FileChunk *pTmp = pChunk; - pChunk = pChunk->pNext; - sqlite3_free(pTmp); + if( p->pReal ){ + return sqlite3OsTruncate(p->pReal, size); + }else if( size==0 ){ + memjrnlFreeChunks(p); + p->nSize = 0; + p->endpoint.pChunk = 0; + p->endpoint.iOffset = 0; + p->readpoint.pChunk = 0; + p->readpoint.iOffset = 0; } - sqlite3MemJournalOpen(pJfd); return SQLITE_OK; } @@ -177,21 +268,23 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ ** Close the file. */ static int memjrnlClose(sqlite3_file *pJfd){ - memjrnlTruncate(pJfd, 0); + MemJournal *p = (MemJournal *)pJfd; + memjrnlFreeChunks(p); + if( p->pReal ) sqlite3OsClose(p->pReal); return SQLITE_OK; } - /* ** Sync the file. ** -** Syncing an in-memory journal is a no-op. And, in fact, this routine -** is never called in a working implementation. This implementation -** exists purely as a contingency, in case some malfunction in some other -** part of SQLite causes Sync to be called by mistake. +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. */ -static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); +static int memjrnlSync(sqlite3_file *pJfd, int flags){ + MemJournal *p = (MemJournal *)pJfd; + if( p->pReal ){ + return sqlite3OsSync(p->pReal, flags); + } return SQLITE_OK; } @@ -200,6 +293,9 @@ static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ */ static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ MemJournal *p = (MemJournal *)pJfd; + if( p->pReal ){ + return sqlite3OsFileSize(p->pReal, pSize); + } *pSize = (sqlite_int64) p->endpoint.iOffset; return SQLITE_OK; } @@ -230,26 +326,92 @@ static const struct sqlite3_io_methods MemJournalMethods = { }; /* -** Open a journal file. +** Open a journal file. +** +** The behaviour of the journal file depends on the value of parameter +** nBuf. If nBuf is 0, then the journal file is always create and +** accessed using the underlying VFS. If nBuf is less than zero, then +** all content is always stored in main-memory. Finally, if nBuf is a +** positive value, then the journal file is initially created in-memory +** but may be flushed to disk later on. In this case the journal file is +** flushed to disk either when it grows larger than nBuf bytes in size, +** or when sqlite3JournalCreate() is called. */ -void sqlite3MemJournalOpen(sqlite3_file *pJfd){ - MemJournal *p = (MemJournal *)pJfd; - assert( EIGHT_BYTE_ALIGNMENT(p) ); - memset(p, 0, sqlite3MemJournalSize()); - p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; +int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nBuf /* Bytes buffered before opening the file */ +){ + MemJournal *p = (MemJournal*)pJfd; + + /* Zero the file-handle object. If nBuf was passed zero, initialize + ** it using the sqlite3OsOpen() function of the underlying VFS. In this + ** case none of the code in this module is executed as a result of calls + ** made on the journal file-handle. */ + memset(p, 0, sizeof(MemJournal)); + if( nBuf==0 ){ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + + if( nBuf>0 ){ + p->nChunkSize = nBuf; + }else{ + p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); + assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); + } + + p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; + p->nBuf = nBuf; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; } /* -** Return true if the file-handle passed as an argument is -** an in-memory journal +** Open an in-memory journal file. +*/ +void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + sqlite3JournalOpen(0, 0, pJfd, 0, -1); +} + +/* +** Return true if the file-handle passed as an argument is an in-memory +** journal. */ int sqlite3IsMemJournal(sqlite3_file *pJfd){ - return pJfd->pMethods==&MemJournalMethods; + return pJfd->pMethods==&MemJournalMethods && ((MemJournal*)pJfd)->pReal==0; +} + +/* +** If the argument p points to a MemJournal structure that is not an +** in-memory-only journal file, and the underlying file has not yet been +** created, create it now. +*/ +int sqlite3JournalCreate(sqlite3_file *p){ + int rc = SQLITE_OK; + if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nBuf>=0 ){ + rc = createFile((MemJournal*)p); + } + return rc; +} + +/* +** The file-handle passed as teh only argument is open on a journal file. +** Return true if this "journal file" is actually stored in heap memory, +** or false otherwise. +*/ +int sqlite3JournalIsInMemory(sqlite3_file *p){ + return p->pMethods==&MemJournalMethods && ((MemJournal*)p)->pReal==0; } /* -** Return the number of bytes required to store a MemJournal file descriptor. +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. */ -int sqlite3MemJournalSize(void){ - return sizeof(MemJournal); +int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return pVfs->szOsFile + sizeof(MemJournal); } + diff --git a/src/os.c b/src/os.c index 2fd44abf85..eed7828538 100644 --- a/src/os.c +++ b/src/os.c @@ -66,7 +66,7 @@ int sqlite3_open_file_count = 0; #if defined(SQLITE_TEST) int sqlite3_memdebug_vfs_oom_test = 1; #define DO_OS_MALLOC_TEST(x) \ - if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3IsMemJournal(x))) { \ + if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \ void *pTstAlloc = sqlite3Malloc(10); \ if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT; \ sqlite3_free(pTstAlloc); \ diff --git a/src/pager.c b/src/pager.c index 51bd45e486..5998a57831 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1724,7 +1724,7 @@ static void releaseAllSavepoints(Pager *pPager){ for(ii=0; iinSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ + if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ sqlite3OsClose(pPager->sjfd); } sqlite3_free(pPager->aSavepoint); @@ -1962,8 +1962,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ - if( sqlite3IsMemJournal(pPager->jfd) ){ - assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + if( sqlite3JournalIsInMemory(pPager->jfd) ){ + /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ sqlite3OsClose(pPager->jfd); }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ if( pPager->journalOff==0 ){ @@ -1991,7 +1991,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ ** file should be closed and deleted. If this connection writes to ** the database file, it will do so using an in-memory journal. */ - int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); + int bDelete = (!pPager->tempFile&&!sqlite3JournalIsInMemory(pPager->jfd)); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL @@ -4578,18 +4578,8 @@ int sqlite3PagerOpen( int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle - ** (there are two of them, the main journal and the sub-journal). This - ** is the maximum space required for an in-memory journal file handle - ** and a regular journal file-handle. Note that a "regular journal-handle" - ** may be a wrapper capable of caching the first portion of the journal - ** file in memory to implement the atomic-write optimization (see - ** source file journal.c). - */ - if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ - journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); - }else{ - journalFileSize = ROUND8(sqlite3MemJournalSize()); - } + ** (there are two of them, the main journal and the sub-journal). */ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; @@ -6667,7 +6657,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ - if( sqlite3IsMemJournal(pPager->sjfd) ){ + if( sqlite3JournalIsInMemory(pPager->sjfd) ){ rc = sqlite3OsTruncate(pPager->sjfd, 0); assert( rc==SQLITE_OK ); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 759d7ca5e1..4cbb7f3d06 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4004,15 +4004,12 @@ int sqlite3FindInIndex(Parse *, Expr *, u32, int*); int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); int sqlite3JournalSize(sqlite3_vfs *); int sqlite3JournalCreate(sqlite3_file *); - int sqlite3JournalExists(sqlite3_file *p); #else #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) - #define sqlite3JournalExists(p) 1 #endif +int sqlite3JournalIsInMemory(sqlite3_file *p); void sqlite3MemJournalOpen(sqlite3_file *); -int sqlite3MemJournalSize(void); -int sqlite3IsMemJournal(sqlite3_file *); void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 From 5f37ed5121b705f6e2683b071a4bf34f66005cb9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Feb 2016 20:00:13 +0000 Subject: [PATCH 2/5] Fix minor problems with new code in memjournal.c. FossilOrigin-Name: 9fd3f7b9c93325a83cdbe7dc9ee312b22a9c2898 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/memjournal.c | 25 +++++++++++-------------- src/pager.c | 8 +++++--- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 7546ccd6d9..97a63c3c19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extend\sthe\scode\sin\smemjournal.c\sso\sthat\sit\ssubsumes\sthe\srole\sof\sjournal.c.\sAnd\s(untested)\scan\sflush\sjournal\sor\sstatement\sjournal\sfiles\sto\sdisk\safter\sthey\sgrow\sto\sa\sspecified\ssize. -D 2016-02-27T20:14:55.339 +C Fix\sminor\sproblems\swith\snew\scode\sin\smemjournal.c. +D 2016-02-29T20:00:13.233 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 4f319afb7c049d40aff7af6e8c4e7cc2ba18e079 @@ -321,7 +321,7 @@ F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memjournal.c 8d045e50120e9988e63bb0af69187e1e5e35122f +F src/memjournal.c 9b71f171dd20827022e9edd874adf8dba2b8d218 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 @@ -336,7 +336,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 3a6f20736dfb8a0949cdd66553fdf59f6604be35 F src/os_win.c f0d7aa603eb6262143d7169a222aea07c4fca91d F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c e2bf43b506447d649e9d9266e988007ae7248699 +F src/pager.c 24dd5e463f527427dd55fccc338c62260ae09b3a F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y c3ce2c4a7cbf0b699239be6b2a945c5cb51875e2 F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df @@ -1451,10 +1451,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f681d800340e0e710f73d0f7c09101f899249183 -R f456c30e495865d50fc2c9c2ddb305ed -T *branch * memjournal-exp -T *sym-memjournal-exp * -T -sym-trunk * +P e0b0b4316531fc8afa74b4882d9c74a91030ec69 +R 662914373e55c4521cf0e8e3f2bf03da U dan -Z dad681f00ca4c98b68e43d3727f3c92e +Z 9ebd4b81ff0d949546d50775b32308aa diff --git a/manifest.uuid b/manifest.uuid index b15150144a..9b400a9c26 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0b0b4316531fc8afa74b4882d9c74a91030ec69 \ No newline at end of file +9fd3f7b9c93325a83cdbe7dc9ee312b22a9c2898 \ No newline at end of file diff --git a/src/memjournal.c b/src/memjournal.c index 6e1bf4edf7..632f9dc023 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -248,6 +248,10 @@ static int memjrnlWrite( /* ** Truncate the file. +** +** If the journal file is already on disk, truncate it there. Or, if it +** is still in main memory but is being truncated to zero bytes in size, +** ignore */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; @@ -350,7 +354,7 @@ int sqlite3JournalOpen( ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls ** made on the journal file-handle. */ - memset(p, 0, sizeof(MemJournal)); + memset(p, 0, sizeof(MemJournal) + pVfs ? pVfs->szOsFile : 0); if( nBuf==0 ){ return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); } @@ -377,30 +381,23 @@ void sqlite3MemJournalOpen(sqlite3_file *pJfd){ sqlite3JournalOpen(0, 0, pJfd, 0, -1); } -/* -** Return true if the file-handle passed as an argument is an in-memory -** journal. -*/ -int sqlite3IsMemJournal(sqlite3_file *pJfd){ - return pJfd->pMethods==&MemJournalMethods && ((MemJournal*)pJfd)->pReal==0; -} - /* ** If the argument p points to a MemJournal structure that is not an -** in-memory-only journal file, and the underlying file has not yet been -** created, create it now. +** in-memory-only journal file (i.e. is one that was opened with a +ve +** nBuf parameter), and the underlying file has not yet been created, +** create it now. */ int sqlite3JournalCreate(sqlite3_file *p){ int rc = SQLITE_OK; - if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nBuf>=0 ){ + if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nBuf>0 ){ rc = createFile((MemJournal*)p); } return rc; } /* -** The file-handle passed as teh only argument is open on a journal file. -** Return true if this "journal file" is actually stored in heap memory, +** The file-handle passed as the only argument is open on a journal file. +** Return true if this "journal file" is currently stored in heap memory, ** or false otherwise. */ int sqlite3JournalIsInMemory(sqlite3_file *p){ diff --git a/src/pager.c b/src/pager.c index 5998a57831..81c6572c9d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1343,6 +1343,7 @@ static i64 journalHdrOffset(Pager *pPager){ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); + assert( !sqlite3JournalIsInMemory(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ @@ -1989,9 +1990,10 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. + ** the database file, it will do so using an in-memory journal. */ - int bDelete = (!pPager->tempFile&&!sqlite3JournalIsInMemory(pPager->jfd)); + int bDelete = !pPager->tempFile; + assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL @@ -2729,7 +2731,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, - ** mxPathname is 512, which is the same as the minimum allowable value + ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ zMaster = pPager->pTmpSpace; From 6e76326d185bc37d386a25181fe275fe3fd7b18e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Feb 2016 20:18:21 +0000 Subject: [PATCH 3/5] When using a temporary file for a statement journal, store the first 64KiB in memory. If the file grows larger than that, flush it to disk and free the memory. Hardcoding to 64KiB is just an experiment to check that the memjournal.c code works. FossilOrigin-Name: 44b2dc18e200e87cf062cb8f1659727c53fa36e9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/memjournal.c | 2 +- src/pager.c | 9 ++++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 97a63c3c19..0f2eb5c9d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sproblems\swith\snew\scode\sin\smemjournal.c. -D 2016-02-29T20:00:13.233 +C When\susing\sa\stemporary\sfile\sfor\sa\sstatement\sjournal,\sstore\sthe\sfirst\s64KiB\sin\smemory.\sIf\sthe\sfile\sgrows\slarger\sthan\sthat,\sflush\sit\sto\sdisk\sand\sfree\sthe\smemory.\sHardcoding\sto\s64KiB\sis\sjust\san\sexperiment\sto\scheck\sthat\sthe\smemjournal.c\scode\sworks. +D 2016-02-29T20:18:21.986 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 4f319afb7c049d40aff7af6e8c4e7cc2ba18e079 @@ -321,7 +321,7 @@ F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memjournal.c 9b71f171dd20827022e9edd874adf8dba2b8d218 +F src/memjournal.c fea987c5809993211da554ba109816fae7e04dae F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 @@ -336,7 +336,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 3a6f20736dfb8a0949cdd66553fdf59f6604be35 F src/os_win.c f0d7aa603eb6262143d7169a222aea07c4fca91d F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 24dd5e463f527427dd55fccc338c62260ae09b3a +F src/pager.c 2bc43817697b5a4e88fd6a2cdb2cb25f2223505c F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y c3ce2c4a7cbf0b699239be6b2a945c5cb51875e2 F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df @@ -1451,7 +1451,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e0b0b4316531fc8afa74b4882d9c74a91030ec69 -R 662914373e55c4521cf0e8e3f2bf03da +P 9fd3f7b9c93325a83cdbe7dc9ee312b22a9c2898 +R 498bd0458994e1419985675b963a5c33 U dan -Z 9ebd4b81ff0d949546d50775b32308aa +Z de04134d7998c131f105dcb3752be845 diff --git a/manifest.uuid b/manifest.uuid index 9b400a9c26..f58d5d3e8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fd3f7b9c93325a83cdbe7dc9ee312b22a9c2898 \ No newline at end of file +44b2dc18e200e87cf062cb8f1659727c53fa36e9 \ No newline at end of file diff --git a/src/memjournal.c b/src/memjournal.c index 632f9dc023..eaa4d4f83a 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -354,7 +354,7 @@ int sqlite3JournalOpen( ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls ** made on the journal file-handle. */ - memset(p, 0, sizeof(MemJournal) + pVfs ? pVfs->szOsFile : 0); + memset(p, 0, sizeof(MemJournal) + (pVfs ? pVfs->szOsFile : 0)); if( nBuf==0 ){ return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); } diff --git a/src/pager.c b/src/pager.c index 81c6572c9d..c74748c8e4 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4355,11 +4355,14 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + | SQLITE_OPEN_DELETEONCLOSE; + int nBuf = 64*1024; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ - sqlite3MemJournalOpen(pPager->sjfd); - }else{ - rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + nBuf = -1; } + rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nBuf); } return rc; } From ff6b82663181c0ebdd4b57a0e858466c3658c78d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Mar 2016 00:13:29 +0000 Subject: [PATCH 4/5] Get in-memory subjournals working for builds without SQLITE_ENABLE_ATOMIC_WRITE. FossilOrigin-Name: b5378dcea53d4cd1744d13ddee5e50869607b771 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/memjournal.c | 3 ++- src/sqliteInt.h | 6 ++---- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c50e674c64..9f2d2fdb74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\supdates\sfrom\strunk. -D 2016-03-03T21:29:10.682 +C Get\sin-memory\ssubjournals\sworking\sfor\sbuilds\swithout\sSQLITE_ENABLE_ATOMIC_WRITE. +D 2016-03-04T00:13:29.040 F Makefile.in e335453db0b16da00c884ad51bb56d1c091a74de F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc dbd4621ecc585c2ef0c2aa0874698c54675754f1 @@ -321,7 +321,7 @@ F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memjournal.c fea987c5809993211da554ba109816fae7e04dae +F src/memjournal.c 56c7688565cf407c5f22f03e3be478d3a92bdc81 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 @@ -354,7 +354,7 @@ F src/shell.c 5e0ab1e708dc294330ccd8230536e1801f60822e F src/sqlite.h.in 57d2a02b14c9ec4f7cb294153eaf62294dc5aa68 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h 57da6fd6aa35dff72eb946064ff4999eab6bc616 +F src/sqliteInt.h 8a4810fa90b17eb08acab43649b877e894ea1e98 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -1453,7 +1453,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 44b2dc18e200e87cf062cb8f1659727c53fa36e9 16fbf2e19c22df9441aef5c8b7f5670adc38a6b6 -R bdaefcabe5bf87c04d8b11c17326d65d +P 55c00f716dc98b188c91f3a5a010242c9497785f +R 9e3fbc3007cf6215e63dd7896623508e U drh -Z 002d922341500f2c0dc185f71a114083 +Z 727109808ff36a69d33c43dee6d82c50 diff --git a/manifest.uuid b/manifest.uuid index 5a1d0fe44a..7f1e0b5ee4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55c00f716dc98b188c91f3a5a010242c9497785f \ No newline at end of file +b5378dcea53d4cd1744d13ddee5e50869607b771 \ No newline at end of file diff --git a/src/memjournal.c b/src/memjournal.c index eaa4d4f83a..9ecd2a9ba3 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -381,6 +381,7 @@ void sqlite3MemJournalOpen(sqlite3_file *pJfd){ sqlite3JournalOpen(0, 0, pJfd, 0, -1); } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE /* ** If the argument p points to a MemJournal structure that is not an ** in-memory-only journal file (i.e. is one that was opened with a +ve @@ -394,6 +395,7 @@ int sqlite3JournalCreate(sqlite3_file *p){ } return rc; } +#endif /* ** The file-handle passed as the only argument is open on a journal file. @@ -411,4 +413,3 @@ int sqlite3JournalIsInMemory(sqlite3_file *p){ int sqlite3JournalSize(sqlite3_vfs *pVfs){ return pVfs->szOsFile + sizeof(MemJournal); } - diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 69cbb6c604..387bb7436b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3998,12 +3998,10 @@ const char *sqlite3JournalModename(int); #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ int sqlite3FindInIndex(Parse *, Expr *, u32, int*); +int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); +int sqlite3JournalSize(sqlite3_vfs *); #ifdef SQLITE_ENABLE_ATOMIC_WRITE - int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); - int sqlite3JournalSize(sqlite3_vfs *); int sqlite3JournalCreate(sqlite3_file *); -#else - #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) #endif int sqlite3JournalIsInMemory(sqlite3_file *p); From 3ac9a86415ad92af53f10964b6b522ef96c03afb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Mar 2016 14:23:10 +0000 Subject: [PATCH 5/5] Update test cases to taken deferred statement-journal opening into account. FossilOrigin-Name: 5b2fe5219ab9ad15969e3374894a75979b65cb3c --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/conflict.test | 12 +++++++----- test/conflict2.test | 23 +++++++++++++---------- test/exclusive.test | 9 ++++++--- test/stmt.test | 10 ++++++---- test/tempdb.test | 2 +- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 58b8efaa39..b827dc21f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\schanges\sfrom\strunk. -D 2016-03-04T13:08:35.636 +C Update\stest\scases\sto\staken\sdeferred\sstatement-journal\sopening\sinto\saccount. +D 2016-03-04T14:23:10.937 F Makefile.in ead489fe5d3313b8a2a8d5f3710173a4b50dd2e0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d68db8ddac21ea2576ea5b446f12cf9ebe897a03 @@ -541,8 +541,8 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b -F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 -F test/conflict2.test a82dd3b9b41fceb5dd6ff0707c5c7ffba208d538 +F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db +F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 F test/corrupt.test 141c39ea650c1365e85a49e402fa05cb9617fb97 @@ -627,7 +627,7 @@ F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 85873fa5816c48915c82c4e74cb5c35a5b48160f F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c -F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 +F test/exclusive.test f48243eaf40e0957215501a12f510a8644d13a02 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 @@ -1063,7 +1063,7 @@ F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a F test/stat.test acc91e80517fff447ae8adcfd953cfdaa5efc0af F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 -F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 +F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 @@ -1079,7 +1079,7 @@ F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 7fb866443c7deceed22b63948ccd6f76b52ad054 -F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c +F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 750365ff97047ded5f2d6e28df82a998f7c66ae0 @@ -1453,7 +1453,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b5378dcea53d4cd1744d13ddee5e50869607b771 2682e8e413fadbca0673f242769af17bfd291559 -R 1d5f238c78a751f41b8ce4e327b6fc98 +P a87305dfd49ee17632d0bce66ce7c2e9407b7b3e +R bcfeac6deb0b24b3190e8b427e406c58 U drh -Z 35e5851abb6d53f81aa001148a407eb0 +Z 82bd4288d2e4175a3c31e21a46f9d82d diff --git a/manifest.uuid b/manifest.uuid index 7ae252503c..5596275a80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a87305dfd49ee17632d0bce66ce7c2e9407b7b3e \ No newline at end of file +5b2fe5219ab9ad15969e3374894a75979b65cb3c \ No newline at end of file diff --git a/test/conflict.test b/test/conflict.test index af5668ed72..a39988adb0 100644 --- a/test/conflict.test +++ b/test/conflict.test @@ -285,15 +285,17 @@ do_test conflict-6.0 { # t3 Number of temporary files for tables # t4 Number of temporary files for statement journals # -# Update: Since temporary table files are now opened lazily, and none -# of the following tests use large quantities of data, t3 is always 0. +# Update (2007-08-21): Since temporary table files are now opened lazily, +# and none of the following tests use large quantities of data, t3 is always 0. +# +# Update (2016-03-04): Subjournals now also open lazily, so t4 is also always 0. # foreach {i conf1 cmd t0 t1 t2 t3 t4} { - 1 {} UPDATE 1 {6 7 8 9} 1 0 1 + 1 {} UPDATE 1 {6 7 8 9} 1 0 0 2 REPLACE UPDATE 0 {7 6 9} 1 0 0 3 IGNORE UPDATE 0 {6 7 3 9} 1 0 0 4 FAIL UPDATE 1 {6 7 3 4} 1 0 0 - 5 ABORT UPDATE 1 {1 2 3 4} 1 0 1 + 5 ABORT UPDATE 1 {1 2 3 4} 1 0 0 6 ROLLBACK UPDATE 1 {1 2 3 4} 0 0 0 7 REPLACE {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 @@ -303,7 +305,7 @@ foreach {i conf1 cmd t0 t1 t2 t3 t4} { 12 {} {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 14 {} {UPDATE OR FAIL} 1 {6 7 3 4} 1 0 0 - 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 0 1 + 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 0 0 16 {} {UPDATE OR ROLLBACK} 1 {1 2 3 4} 0 0 0 } { if {$t0} {set t1 {UNIQUE constraint failed: t1.a}} diff --git a/test/conflict2.test b/test/conflict2.test index 6496913849..f33e01d582 100644 --- a/test/conflict2.test +++ b/test/conflict2.test @@ -287,22 +287,25 @@ do_test conflict2-6.0 { # Update: Since temporary table files are now opened lazily, and none # of the following tests use large quantities of data, t3 is always 0. # +# Update (2016-03-04): Subjournals now only open when their size +# exceeds 64KB. +# foreach {i conf1 cmd t0 t1 t2 t3 t4} { - 1 {} UPDATE 1 {6 7 8 9} 1 0 1 - 2 REPLACE UPDATE 0 {7 6 9} 1 0 1 - 3 IGNORE UPDATE 0 {6 7 3 9} 1 0 1 - 4 FAIL UPDATE 1 {6 7 3 4} 1 0 1 - 5 ABORT UPDATE 1 {1 2 3 4} 1 0 1 - 6 ROLLBACK UPDATE 1 {1 2 3 4} 0 0 1 + 1 {} UPDATE 1 {6 7 8 9} 1 0 0 + 2 REPLACE UPDATE 0 {7 6 9} 1 0 0 + 3 IGNORE UPDATE 0 {6 7 3 9} 1 0 0 + 4 FAIL UPDATE 1 {6 7 3 4} 1 0 0 + 5 ABORT UPDATE 1 {1 2 3 4} 1 0 0 + 6 ROLLBACK UPDATE 1 {1 2 3 4} 0 0 0 7 REPLACE {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 - 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 0 1 + 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 9 FAIL {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 - 10 ABORT {UPDATE OR REPLACE} 0 {7 6 9} 1 0 1 + 10 ABORT {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 11 ROLLBACK {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 12 {} {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 - 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 0 1 + 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 14 {} {UPDATE OR FAIL} 1 {6 7 3 4} 1 0 0 - 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 0 1 + 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 0 0 16 {} {UPDATE OR ROLLBACK} 1 {1 2 3 4} 0 0 0 } { diff --git a/test/exclusive.test b/test/exclusive.test index c000dfefa4..c7b88cfbca 100644 --- a/test/exclusive.test +++ b/test/exclusive.test @@ -420,9 +420,10 @@ do_test exclusive-5.0 { } {} do_test exclusive-5.1 { # Three files are open: The db, journal and statement-journal. + # (2016-03-04) The statement-journal is now opened lazily set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds -} [expr 3 - ($TEMP_STORE>=2)] +} [expr 2 - ($TEMP_STORE>=2)] do_test exclusive-5.2 { execsql { COMMIT; @@ -446,17 +447,19 @@ do_test exclusive-5.4 { INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc; } # Three files are open: The db, journal and statement-journal. + # 2016-03-04: The statement-journal open is deferred set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds -} [expr 3 - ($TEMP_STORE>=2)] +} [expr 2 - ($TEMP_STORE>=2)] do_test exclusive-5.5 { execsql { COMMIT; } # Three files are still open: The db, journal and statement-journal. + # 2016-03-04: The statement-journal open is deferred set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds -} [expr 3 - ($TEMP_STORE>=2)] +} [expr 2 - ($TEMP_STORE>=2)] do_test exclusive-5.6 { execsql { PRAGMA locking_mode = normal; diff --git a/test/stmt.test b/test/stmt.test index 49a41414b7..df501f7654 100644 --- a/test/stmt.test +++ b/test/stmt.test @@ -46,7 +46,8 @@ do_test stmt-1.4 { INSERT INTO t1 SELECT a+1, b+1 FROM t1; } set sqlite_open_file_count -} {3} + # 2016-03-04: statement-journal open deferred +} {2} do_test stmt-1.5 { execsql COMMIT set sqlite_open_file_count @@ -61,7 +62,8 @@ do_test stmt-1.6.1 { do_test stmt-1.6.2 { execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 } set sqlite_open_file_count -} {3} + # 2016-03-04: statement-journal open deferred +} {2} do_test stmt-1.7 { execsql COMMIT set sqlite_open_file_count @@ -84,7 +86,7 @@ filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9 } 2 filecount stmt-2.4 { INSERT INTO t1 SELECT 9, 9; INSERT INTO t1 SELECT 10, 10; -} 3 +} 2 do_test stmt-2.5 { execsql { CREATE INDEX i1 ON t1(b) } @@ -92,6 +94,6 @@ do_test stmt-2.5 { filecount stmt-2.6 { REPLACE INTO t1 VALUES(5, 5); REPLACE INTO t1 VALUES(5, 5); -} 3 +} 2 finish_test diff --git a/test/tempdb.test b/test/tempdb.test index ecd23d4e32..61416ec81c 100644 --- a/test/tempdb.test +++ b/test/tempdb.test @@ -76,7 +76,7 @@ do_test tempdb-2.2 { } catchsql { INSERT INTO t1 SELECT * FROM t2 } set sqlite_open_file_count -} [expr 1 + (0==$jrnl_in_memory) + (0==$subj_in_memory)] +} [expr 1 + (0==$jrnl_in_memory)] do_test tempdb-2.3 { execsql { PRAGMA temp_store = 'memory';