Defer opening and writing statement journals until the size reaches a

threshold (currently 64KiB).

FossilOrigin-Name: cb9302cca423de41305719a49208daa392ec09da

manifest

@@ -1,5 +1,5 @@
-C Change\sthe\sdefault\scache_size\sto\s-2000\s(which\smeans\s2000*1024\sbytes\nindependent\sof\spage_size).
+C Defer\sopening\sand\swriting\sstatement\sjournals\suntil\sthe\ssize\sreaches\sa\nthreshold\s(currently\s64KiB).
-D 2016-03-04T04:01:43.190
+D 2016-03-04T14:43:44.681
 F Makefile.in ead489fe5d3313b8a2a8d5f3710173a4b50dd2e0
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d68db8ddac21ea2576ea5b446f12cf9ebe897a03
@@ -311,7 +311,7 @@ F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
 F src/insert.c 723d5d708cdb61bdd47c00b9f07c75be45aefc09
-F src/journal.c fe3a3e2559ce3ce9d371afd30fbabbc074174575
+F src/journal.c 673cbdde5676eb0c55848f561575d45b609c820d
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
 F src/loadext.c 9e2a41adcaff16ebc1ebff1f336cbf33de55396f
 F src/main.c 32c45647866429f34d7a13a717172a8e7d0a1056
@@ -321,7 +321,7 @@ F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
-F src/memjournal.c 349a04fb803db40532cde0993e39758f1acaecce
+F src/memjournal.c 56c7688565cf407c5f22f03e3be478d3a92bdc81
 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495
 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
@@ -329,14 +329,14 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
 F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
-F src/os.c ca9a104b677328ee037cfdf1a54a16fd1805e8c9
+F src/os.c f89e3ca1c2e3d5015b847aec60371c474acbac82
 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c f5bac8e74aaefc4ea520e43b4540793c3b8a9e8f
 F src/os_win.c f0d7aa603eb6262143d7169a222aea07c4fca91d
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
-F src/pager.c d034c69b958c01289eb8070cbf902e1a68cd7e0b
+F src/pager.c 2bc43817697b5a4e88fd6a2cdb2cb25f2223505c
 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56
 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821
 F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df
@@ -354,7 +354,7 @@ F src/shell.c 5e0ab1e708dc294330ccd8230536e1801f60822e
 F src/sqlite.h.in 57d2a02b14c9ec4f7cb294153eaf62294dc5aa68
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
-F src/sqliteInt.h 8aa9992cb35289ede0946f7ba8e993070e040fc4
+F src/sqliteInt.h 8a4810fa90b17eb08acab43649b877e894ea1e98
 F src/sqliteLimit.h 7b28cf72cbd52f178bfc97ea266445e351f2cd24
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9
@@ -541,8 +541,8 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b
-F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8
+F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
-F test/conflict2.test a82dd3b9b41fceb5dd6ff0707c5c7ffba208d538
+F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b
 F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4
 F test/corrupt.test 141c39ea650c1365e85a49e402fa05cb9617fb97
@@ -627,7 +627,7 @@ F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020
 F test/eqp.test 85873fa5816c48915c82c4e74cb5c35a5b48160f
 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401
 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c
-F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75
+F test/exclusive.test f48243eaf40e0957215501a12f510a8644d13a02
 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30
@@ -1063,7 +1063,7 @@ F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
 F test/stat.test acc91e80517fff447ae8adcfd953cfdaa5efc0af
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
-F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9
+F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f
 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
 F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f
 F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4
@@ -1079,7 +1079,7 @@ F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
 F test/tclsqlite.test 7fb866443c7deceed22b63948ccd6f76b52ad054
-F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c
+F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6
 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1
 F test/tester.tcl 750365ff97047ded5f2d6e28df82a998f7c66ae0
@@ -1453,7 +1453,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2e8a9ca9d3d6efddc31f01074d14acae56568c37
+P 2682e8e413fadbca0673f242769af17bfd291559 5b2fe5219ab9ad15969e3374894a75979b65cb3c
-R bce4d7ea1e35eefbc9f3bc83f2294e5c
+R bcfeac6deb0b24b3190e8b427e406c58
+T +closed 5b2fe5219ab9ad15969e3374894a75979b65cb3c
 U drh
-Z 8be75f8161fe6fe960ec420b19540c0b
+Z 04f483c25715ded0ed1694d083bfc434

manifest.uuid

@@ -1 +1 @@
-2682e8e413fadbca0673f242769af17bfd291559
+cb9302cca423de41305719a49208daa392ec09da

src/journal.c

@@ -24,6 +24,7 @@
 **      buffer, or
 **   2) The sqlite3JournalCreate() function is called.
 */
+#if 0 
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
 #include "sqliteInt.h"
 
@@ -254,3 +255,4 @@ int sqlite3JournalSize(sqlite3_vfs *pVfs){
   return (pVfs->szOsFile+sizeof(JournalFile));
 }
 #endif
+#endif

src/memjournal.c

@@ -21,24 +21,28 @@ typedef struct MemJournal MemJournal;
 typedef struct FilePoint FilePoint;
 typedef struct FileChunk FileChunk;
 
-/* Space to hold the rollback journal is allocated in increments of
-** this many bytes.
-**
-** The size chosen is a little less than a power of two.  That way,
-** the FileChunk object will have a size that almost exactly fills
-** a power-of-two allocation.  This minimizes wasted space in power-of-two
-** memory allocators.
-*/
-#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*)))
-
 /*
 ** The rollback journal is composed of a linked list of these structures.
+**
+** The zChunk array is always at least 8 bytes in size - usually much more.
+** Its actual size is stored in the MemJournal.nChunkSize variable.
 */
 struct FileChunk {
   FileChunk *pNext;               /* Next chunk in the journal */
-  u8 zChunk[JOURNAL_CHUNKSIZE];   /* Content of this chunk */
+  u8 zChunk[8];                   /* Content of this chunk */
 };
 
+/*
+** By default, allocate this many bytes of memory for each FileChunk object.
+*/
+#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024
+
+/*
+** For chunk size nChunkSize, return the number of bytes that should
+** be allocated for each FileChunk structure.
+*/
+#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8))
+
 /*
 ** An instance of this object serves as a cursor into the rollback journal.
 ** The cursor can be either for reading or writing.
@@ -49,14 +53,23 @@ struct FilePoint {
 };
 
 /*
-** This subclass is a subclass of sqlite3_file.  Each open memory-journal
+** This structure is a subclass of sqlite3_file. Each open memory-journal
 ** is an instance of this class.
 */
 struct MemJournal {
-  sqlite3_io_methods *pMethod;    /* Parent class. MUST BE FIRST */
+  const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */
+  int nChunkSize;                 /* In-memory chunk-size */
+
+  int nBuf;                       /* Bytes of data before flushing */
+  int nSize;                      /* Bytes of data currently in memory */
   FileChunk *pFirst;              /* Head of in-memory chunk-list */
   FilePoint endpoint;             /* Pointer to the end of the file */
   FilePoint readpoint;            /* Pointer to the end of the last xRead() */
+
+  int flags;                      /* xOpen flags */
+  sqlite3_vfs *pVfs;              /* The "real" underlying VFS */
+  const char *zJournal;           /* Name of the journal file */
+  sqlite3_file *pReal;            /* The "real" underlying file descriptor */
 };
 
 /*
@@ -70,41 +83,96 @@ static int memjrnlRead(
   sqlite_int64 iOfst     /* Begin reading at this offset */
 ){
   MemJournal *p = (MemJournal *)pJfd;
-  u8 *zOut = zBuf;
+  if( p->pReal ){
-  int nRead = iAmt;
+    return sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst);
-  int iChunkOffset;
+  }else if( (iAmt+iOfst)>p->endpoint.iOffset ){
-  FileChunk *pChunk;
+    return SQLITE_IOERR_SHORT_READ;
-
-  /* SQLite never tries to read past the end of a rollback journal file */
-  assert( iOfst+iAmt<=p->endpoint.iOffset );
-
-  if( p->readpoint.iOffset!=iOfst || iOfst==0 ){
-    sqlite3_int64 iOff = 0;
-    for(pChunk=p->pFirst; 
-        ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst;
-        pChunk=pChunk->pNext
-    ){
-      iOff += JOURNAL_CHUNKSIZE;
-    }
   }else{
-    pChunk = p->readpoint.pChunk;
+    u8 *zOut = zBuf;
-  }
+    int nRead = iAmt;
+    int iChunkOffset;
+    FileChunk *pChunk;
 
-  iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE);
+    if( p->readpoint.iOffset!=iOfst || iOfst==0 ){
-  do {
+      sqlite3_int64 iOff = 0;
-    int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset;
+      for(pChunk=p->pFirst; 
-    int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset));
+          ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst;
-    memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy);
+          pChunk=pChunk->pNext
-    zOut += nCopy;
+      ){
-    nRead -= iSpace;
+        iOff += p->nChunkSize;
-    iChunkOffset = 0;
+      }
-  } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 );
+    }else{
-  p->readpoint.iOffset = iOfst+iAmt;
+      pChunk = p->readpoint.pChunk;
-  p->readpoint.pChunk = pChunk;
+    }
+
+    iChunkOffset = (int)(iOfst%p->nChunkSize);
+    do {
+      int iSpace = p->nChunkSize - iChunkOffset;
+      int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset));
+      memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy);
+      zOut += nCopy;
+      nRead -= iSpace;
+      iChunkOffset = 0;
+    } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 );
+    p->readpoint.iOffset = iOfst+iAmt;
+    p->readpoint.pChunk = pChunk;
+  }
 
   return SQLITE_OK;
 }
 
+/*
+** Free the list of FileChunk structures headed at MemJournal.pFirst.
+*/
+static void memjrnlFreeChunks(MemJournal *p){
+  FileChunk *pIter;
+  FileChunk *pNext;
+  for(pIter=p->pFirst; pIter; pIter=pNext){
+    pNext = pIter->pNext;
+    sqlite3_free(pIter);
+  } 
+  p->pFirst = 0;
+}
+
+/*
+** Flush the contents of memory to a real file on disk.
+*/
+static int createFile(MemJournal *p){
+  int rc = SQLITE_OK;
+  if( !p->pReal ){
+    sqlite3_file *pReal = (sqlite3_file *)&p[1];
+    rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0);
+    if( rc==SQLITE_OK ){
+      int nChunk = p->nChunkSize;
+      i64 iOff = 0;
+      FileChunk *pIter;
+      p->pReal = pReal;
+      for(pIter=p->pFirst; pIter && rc==SQLITE_OK; pIter=pIter->pNext){
+        int nWrite = nChunk;
+        if( pIter==p->endpoint.pChunk ){
+          nWrite = p->endpoint.iOffset % p->nChunkSize;
+          if( nWrite==0 ) nWrite = p->nChunkSize;
+        }
+        rc = sqlite3OsWrite(pReal, pIter->zChunk, nWrite, iOff);
+        iOff += nWrite;
+      }
+      if( rc!=SQLITE_OK ){
+        /* If an error occurred while writing to the file, close it before
+        ** returning. This way, SQLite uses the in-memory journal data to 
+        ** roll back changes made to the internal page-cache before this
+        ** function was called.  */
+        sqlite3OsClose(pReal);
+        p->pReal = 0;
+      }else{
+        /* No error has occurred. Free the in-memory buffers. */
+        memjrnlFreeChunks(p);
+      }
+    }
+  }
+  return rc;
+}
+
+
 /*
 ** Write data to the file.
 */
@@ -118,38 +186,61 @@ static int memjrnlWrite(
   int nWrite = iAmt;
   u8 *zWrite = (u8 *)zBuf;
 
-  /* An in-memory journal file should only ever be appended to. Random
+  /* If the file has already been created on disk. */
-  ** access writes are not required by sqlite.
+  if( p->pReal ){
-  */
+    return sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst);
-  assert( iOfst==p->endpoint.iOffset );
+  }
-  UNUSED_PARAMETER(iOfst);
 
-  while( nWrite>0 ){
+  /* If the file should be created now. */
-    FileChunk *pChunk = p->endpoint.pChunk;
+  else if( p->nBuf>0 && (iAmt+iOfst)>p->nBuf ){
-    int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE);
+    int rc = createFile(p);
-    int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset);
+    if( rc==SQLITE_OK ){
-
+      rc = memjrnlWrite(pJfd, zBuf, iAmt, iOfst);
-    if( iChunkOffset==0 ){
-      /* New chunk is required to extend the file. */
-      FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk));
-      if( !pNew ){
-        return SQLITE_IOERR_NOMEM_BKPT;
-      }
-      pNew->pNext = 0;
-      if( pChunk ){
-        assert( p->pFirst );
-        pChunk->pNext = pNew;
-      }else{
-        assert( !p->pFirst );
-        p->pFirst = pNew;
-      }
-      p->endpoint.pChunk = pNew;
     }
+    return rc;
+  }
 
-    memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace);
+  /* If the contents of this write should be stored in memory */
-    zWrite += iSpace;
+  else{
-    nWrite -= iSpace;
+    /* An in-memory journal file should only ever be appended to. Random
-    p->endpoint.iOffset += iSpace;
+    ** access writes are not required. The only exception to this is when
+    ** the in-memory journal is being used by a connection using the
+    ** atomic-write optimization. In this case the first 28 bytes of the
+    ** journal file may be written as part of committing the transaction. */ 
+    assert( iOfst==p->endpoint.iOffset || iOfst==0 );
+    if( iOfst==0 && p->pFirst ){
+      assert( p->nChunkSize>iAmt );
+      memcpy(p->pFirst->zChunk, zBuf, iAmt);
+    }else{
+      while( nWrite>0 ){
+        FileChunk *pChunk = p->endpoint.pChunk;
+        int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize);
+        int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset);
+
+        if( iChunkOffset==0 ){
+          /* New chunk is required to extend the file. */
+          FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize));
+          if( !pNew ){
+            return SQLITE_IOERR_NOMEM_BKPT;
+          }
+          pNew->pNext = 0;
+          if( pChunk ){
+            assert( p->pFirst );
+            pChunk->pNext = pNew;
+          }else{
+            assert( !p->pFirst );
+            p->pFirst = pNew;
+          }
+          p->endpoint.pChunk = pNew;
+        }
+
+        memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace);
+        zWrite += iSpace;
+        nWrite -= iSpace;
+        p->endpoint.iOffset += iSpace;
+      }
+      p->nSize = iAmt + iOfst;
+    }
   }
 
   return SQLITE_OK;
@@ -157,19 +248,23 @@ static int memjrnlWrite(
 
 /*
 ** Truncate the file.
+**
+** If the journal file is already on disk, truncate it there. Or, if it
+** is still in main memory but is being truncated to zero bytes in size,
+** ignore 
 */
 static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){
   MemJournal *p = (MemJournal *)pJfd;
-  FileChunk *pChunk;
+  if( p->pReal ){
-  assert(size==0);
+    return sqlite3OsTruncate(p->pReal, size);
-  UNUSED_PARAMETER(size);
+  }else if( size==0 ){
-  pChunk = p->pFirst;
+    memjrnlFreeChunks(p);
-  while( pChunk ){
+    p->nSize = 0;
-    FileChunk *pTmp = pChunk;
+    p->endpoint.pChunk = 0;
-    pChunk = pChunk->pNext;
+    p->endpoint.iOffset = 0;
-    sqlite3_free(pTmp);
+    p->readpoint.pChunk = 0;
+    p->readpoint.iOffset = 0;
   }
-  sqlite3MemJournalOpen(pJfd);
   return SQLITE_OK;
 }
 
@@ -177,21 +272,23 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){
 ** Close the file.
 */
 static int memjrnlClose(sqlite3_file *pJfd){
-  memjrnlTruncate(pJfd, 0);
+  MemJournal *p = (MemJournal *)pJfd;
+  memjrnlFreeChunks(p);
+  if( p->pReal ) sqlite3OsClose(p->pReal);
   return SQLITE_OK;
 }
 
-
 /*
 ** Sync the file.
 **
-** Syncing an in-memory journal is a no-op.  And, in fact, this routine
+** If the real file has been created, call its xSync method. Otherwise, 
-** is never called in a working implementation.  This implementation
+** syncing an in-memory journal is a no-op. 
-** exists purely as a contingency, in case some malfunction in some other
-** part of SQLite causes Sync to be called by mistake.
 */
-static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){
+static int memjrnlSync(sqlite3_file *pJfd, int flags){
-  UNUSED_PARAMETER2(NotUsed, NotUsed2);
+  MemJournal *p = (MemJournal *)pJfd;
+  if( p->pReal ){
+    return sqlite3OsSync(p->pReal, flags);
+  }
   return SQLITE_OK;
 }
 
@@ -200,6 +297,9 @@ static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){
 */
 static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){
   MemJournal *p = (MemJournal *)pJfd;
+  if( p->pReal ){
+    return sqlite3OsFileSize(p->pReal, pSize);
+  }
   *pSize = (sqlite_int64) p->endpoint.iOffset;
   return SQLITE_OK;
 }
@@ -230,26 +330,86 @@ static const struct sqlite3_io_methods MemJournalMethods = {
 };
 
 /* 
-** Open a journal file.
+** Open a journal file. 
+**
+** The behaviour of the journal file depends on the value of parameter 
+** nBuf. If nBuf is 0, then the journal file is always create and 
+** accessed using the underlying VFS. If nBuf is less than zero, then
+** all content is always stored in main-memory. Finally, if nBuf is a
+** positive value, then the journal file is initially created in-memory
+** but may be flushed to disk later on. In this case the journal file is
+** flushed to disk either when it grows larger than nBuf bytes in size,
+** or when sqlite3JournalCreate() is called.
 */
-void sqlite3MemJournalOpen(sqlite3_file *pJfd){
+int sqlite3JournalOpen(
-  MemJournal *p = (MemJournal *)pJfd;
+  sqlite3_vfs *pVfs,         /* The VFS to use for actual file I/O */
-  assert( EIGHT_BYTE_ALIGNMENT(p) );
+  const char *zName,         /* Name of the journal file */
-  memset(p, 0, sqlite3MemJournalSize());
+  sqlite3_file *pJfd,        /* Preallocated, blank file handle */
-  p->pMethod = (sqlite3_io_methods*)&MemJournalMethods;
+  int flags,                 /* Opening flags */
+  int nBuf                   /* Bytes buffered before opening the file */
+){
+  MemJournal *p = (MemJournal*)pJfd;
+
+  /* Zero the file-handle object. If nBuf was passed zero, initialize
+  ** it using the sqlite3OsOpen() function of the underlying VFS. In this
+  ** case none of the code in this module is executed as a result of calls
+  ** made on the journal file-handle.  */
+  memset(p, 0, sizeof(MemJournal) + (pVfs ? pVfs->szOsFile : 0));
+  if( nBuf==0 ){
+    return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0);
+  }
+
+  if( nBuf>0 ){
+    p->nChunkSize = nBuf;
+  }else{
+    p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk);
+    assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) );
+  }
+
+  p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods;
+  p->nBuf = nBuf;
+  p->flags = flags;
+  p->zJournal = zName;
+  p->pVfs = pVfs;
+  return SQLITE_OK;
 }
 
 /*
-** Return true if the file-handle passed as an argument is 
+** Open an in-memory journal file.
-** an in-memory journal 
 */
-int sqlite3IsMemJournal(sqlite3_file *pJfd){
+void sqlite3MemJournalOpen(sqlite3_file *pJfd){
-  return pJfd->pMethods==&MemJournalMethods;
+  sqlite3JournalOpen(0, 0, pJfd, 0, -1);
+}
+
+#ifdef SQLITE_ENABLE_ATOMIC_WRITE
+/*
+** If the argument p points to a MemJournal structure that is not an 
+** in-memory-only journal file (i.e. is one that was opened with a +ve
+** nBuf parameter), and the underlying file has not yet been created, 
+** create it now.
+*/
+int sqlite3JournalCreate(sqlite3_file *p){
+  int rc = SQLITE_OK;
+  if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nBuf>0 ){
+    rc = createFile((MemJournal*)p);
+  }
+  return rc;
+}
+#endif
+
+/*
+** The file-handle passed as the only argument is open on a journal file.
+** Return true if this "journal file" is currently stored in heap memory,
+** or false otherwise.
+*/
+int sqlite3JournalIsInMemory(sqlite3_file *p){
+  return p->pMethods==&MemJournalMethods && ((MemJournal*)p)->pReal==0;
 }
 
 /* 
-** Return the number of bytes required to store a MemJournal file descriptor.
+** Return the number of bytes required to store a JournalFile that uses vfs
+** pVfs to create the underlying on-disk files.
 */
-int sqlite3MemJournalSize(void){
+int sqlite3JournalSize(sqlite3_vfs *pVfs){
-  return sizeof(MemJournal);
+  return pVfs->szOsFile + sizeof(MemJournal);
 }

src/os.c

@@ -66,7 +66,7 @@ int sqlite3_open_file_count = 0;
 #if defined(SQLITE_TEST)
 int sqlite3_memdebug_vfs_oom_test = 1;
   #define DO_OS_MALLOC_TEST(x)                                       \
-  if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3IsMemJournal(x))) {  \
+  if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \
     void *pTstAlloc = sqlite3Malloc(10);                             \
     if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT;                  \
     sqlite3_free(pTstAlloc);                                         \

src/pager.c

@@ -1343,6 +1343,7 @@ static i64 journalHdrOffset(Pager *pPager){
 static int zeroJournalHdr(Pager *pPager, int doTruncate){
   int rc = SQLITE_OK;                               /* Return code */
   assert( isOpen(pPager->jfd) );
+  assert( !sqlite3JournalIsInMemory(pPager->jfd) );
   if( pPager->journalOff ){
     const i64 iLimit = pPager->journalSizeLimit;    /* Local cache of jsl */
 
@@ -1724,7 +1725,7 @@ static void releaseAllSavepoints(Pager *pPager){
   for(ii=0; ii<pPager->nSavepoint; ii++){
     sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint);
   }
-  if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){
+  if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){
     sqlite3OsClose(pPager->sjfd);
   }
   sqlite3_free(pPager->aSavepoint);
@@ -1962,8 +1963,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
     assert( !pagerUseWal(pPager) );
 
     /* Finalize the journal file. */
-    if( sqlite3IsMemJournal(pPager->jfd) ){
+    if( sqlite3JournalIsInMemory(pPager->jfd) ){
-      assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY );
+      /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */
       sqlite3OsClose(pPager->jfd);
     }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){
       if( pPager->journalOff==0 ){
@@ -1989,9 +1990,10 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
       /* This branch may be executed with Pager.journalMode==MEMORY if
       ** a hot-journal was just rolled back. In this case the journal
       ** file should be closed and deleted. If this connection writes to
-      ** the database file, it will do so using an in-memory journal. 
+      ** the database file, it will do so using an in-memory journal.
       */
-      int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd));
+      int bDelete = !pPager->tempFile;
+      assert( sqlite3JournalIsInMemory(pPager->jfd)==0 );
       assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE 
            || pPager->journalMode==PAGER_JOURNALMODE_MEMORY 
            || pPager->journalMode==PAGER_JOURNALMODE_WAL 
@@ -2729,7 +2731,7 @@ static int pager_playback(Pager *pPager, int isHot){
   ** TODO: Technically the following is an error because it assumes that
   ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that
   ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c,
-  **  mxPathname is 512, which is the same as the minimum allowable value
+  ** mxPathname is 512, which is the same as the minimum allowable value
   ** for pageSize.
   */
   zMaster = pPager->pTmpSpace;
@@ -4353,11 +4355,14 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){
 static int openSubJournal(Pager *pPager){
   int rc = SQLITE_OK;
   if( !isOpen(pPager->sjfd) ){
+    const int flags =  SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE 
+      | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE 
+      | SQLITE_OPEN_DELETEONCLOSE;
+    int nBuf = 64*1024;
     if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){
-      sqlite3MemJournalOpen(pPager->sjfd);
+      nBuf = -1;
-    }else{
-      rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL);
     }
+    rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nBuf);
   }
   return rc;
 }
@@ -4578,18 +4583,8 @@ int sqlite3PagerOpen(
   int nUri = 0;            /* Number of bytes of URI args at *zUri */
 
   /* Figure out how much space is required for each journal file-handle
-  ** (there are two of them, the main journal and the sub-journal). This
+  ** (there are two of them, the main journal and the sub-journal).  */
-  ** is the maximum space required for an in-memory journal file handle 
+  journalFileSize = ROUND8(sqlite3JournalSize(pVfs));
-  ** and a regular journal file-handle. Note that a "regular journal-handle"
-  ** may be a wrapper capable of caching the first portion of the journal
-  ** file in memory to implement the atomic-write optimization (see 
-  ** source file journal.c).
-  */
-  if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){
-    journalFileSize = ROUND8(sqlite3JournalSize(pVfs));
-  }else{
-    journalFileSize = ROUND8(sqlite3MemJournalSize());
-  }
 
   /* Set the output variable to NULL in case an error occurs. */
   *ppPager = 0;
@@ -6667,7 +6662,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){
     if( op==SAVEPOINT_RELEASE ){
       if( nNew==0 && isOpen(pPager->sjfd) ){
         /* Only truncate if it is an in-memory sub-journal. */
-        if( sqlite3IsMemJournal(pPager->sjfd) ){
+        if( sqlite3JournalIsInMemory(pPager->sjfd) ){
           rc = sqlite3OsTruncate(pPager->sjfd, 0);
           assert( rc==SQLITE_OK );
         }

src/sqliteInt.h

@@ -3998,19 +3998,14 @@ const char *sqlite3JournalModename(int);
 #define IN_INDEX_LOOP        0x0004  /* IN operator used as a loop */
 int sqlite3FindInIndex(Parse *, Expr *, u32, int*);
 
+int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
+int sqlite3JournalSize(sqlite3_vfs *);
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
-  int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
-  int sqlite3JournalSize(sqlite3_vfs *);
   int sqlite3JournalCreate(sqlite3_file *);
-  int sqlite3JournalExists(sqlite3_file *p);
-#else
-  #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile)
-  #define sqlite3JournalExists(p) 1
 #endif
 
+int sqlite3JournalIsInMemory(sqlite3_file *p);
 void sqlite3MemJournalOpen(sqlite3_file *);
-int sqlite3MemJournalSize(void);
-int sqlite3IsMemJournal(sqlite3_file *);
 
 void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p);
 #if SQLITE_MAX_EXPR_DEPTH>0

test/conflict.test

@@ -285,15 +285,17 @@ do_test conflict-6.0 {
 #   t3     Number of temporary files for tables
 #   t4     Number of temporary files for statement journals
 #
-# Update: Since temporary table files are now opened lazily, and none
+# Update (2007-08-21): Since temporary table files are now opened lazily, 
-# of the following tests use large quantities of data, t3 is always 0.
+# and none of the following tests use large quantities of data, t3 is always 0.
+#
+# Update (2016-03-04): Subjournals now also open lazily, so t4 is also always 0.
 #
 foreach {i conf1 cmd t0 t1 t2 t3 t4} {
-  1 {}       UPDATE                  1 {6 7 8 9}  1 0 1
+  1 {}       UPDATE                  1 {6 7 8 9}  1 0 0
   2 REPLACE  UPDATE                  0 {7 6 9}    1 0 0
   3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 0
   4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 0
-  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 1
+  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 0
   6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 0
   7 REPLACE  {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
   8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
@@ -303,7 +305,7 @@ foreach {i conf1 cmd t0 t1 t2 t3 t4} {
  12 {}       {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
  13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
  14 {}       {UPDATE OR FAIL}        1 {6 7 3 4}  1 0 0
- 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 1
+ 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 0
  16 {}       {UPDATE OR ROLLBACK}    1 {1 2 3 4}  0 0 0
 } {
   if {$t0} {set t1 {UNIQUE constraint failed: t1.a}}

test/conflict2.test

@@ -287,22 +287,25 @@ do_test conflict2-6.0 {
 # Update: Since temporary table files are now opened lazily, and none
 # of the following tests use large quantities of data, t3 is always 0.
 #
+# Update (2016-03-04): Subjournals now only open when their size
+# exceeds 64KB.
+#
 foreach {i conf1 cmd t0 t1 t2 t3 t4} {
-  1 {}       UPDATE                  1 {6 7 8 9}  1 0 1
+  1 {}       UPDATE                  1 {6 7 8 9}  1 0 0
-  2 REPLACE  UPDATE                  0 {7 6 9}    1 0 1
+  2 REPLACE  UPDATE                  0 {7 6 9}    1 0 0
-  3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 1
+  3 IGNORE   UPDATE                  0 {6 7 3 9}  1 0 0
-  4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 1
+  4 FAIL     UPDATE                  1 {6 7 3 4}  1 0 0
-  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 1
+  5 ABORT    UPDATE                  1 {1 2 3 4}  1 0 0
-  6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 1
+  6 ROLLBACK UPDATE                  1 {1 2 3 4}  0 0 0
   7 REPLACE  {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
-  8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 1
+  8 IGNORE   {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
   9 FAIL     {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
- 10 ABORT    {UPDATE OR REPLACE}     0 {7 6 9}    1 0 1
+ 10 ABORT    {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
  11 ROLLBACK {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
  12 {}       {UPDATE OR IGNORE}      0 {6 7 3 9}  1 0 0
- 13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 1
+ 13 {}       {UPDATE OR REPLACE}     0 {7 6 9}    1 0 0
  14 {}       {UPDATE OR FAIL}        1 {6 7 3 4}  1 0 0
- 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 1
+ 15 {}       {UPDATE OR ABORT}       1 {1 2 3 4}  1 0 0
  16 {}       {UPDATE OR ROLLBACK}    1 {1 2 3 4}  0 0 0
 } {
 

test/exclusive.test

@@ -420,9 +420,10 @@ do_test exclusive-5.0 {
 } {}
 do_test exclusive-5.1 {
   # Three files are open: The db, journal and statement-journal.
+  # (2016-03-04) The statement-journal is now opened lazily
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 3 - ($TEMP_STORE>=2)]
+} [expr 2 - ($TEMP_STORE>=2)]
 do_test exclusive-5.2 {
   execsql {
     COMMIT;
@@ -446,17 +447,19 @@ do_test exclusive-5.4 {
     INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc;
   }
   # Three files are open: The db, journal and statement-journal.
+  # 2016-03-04: The statement-journal open is deferred
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 3 - ($TEMP_STORE>=2)]
+} [expr 2 - ($TEMP_STORE>=2)]
 do_test exclusive-5.5 {
   execsql {
     COMMIT;
   }
   # Three files are still open: The db, journal and statement-journal.
+  # 2016-03-04: The statement-journal open is deferred
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 3 - ($TEMP_STORE>=2)]
+} [expr 2 - ($TEMP_STORE>=2)]
 do_test exclusive-5.6 {
   execsql {
     PRAGMA locking_mode = normal;

test/stmt.test

@@ -46,7 +46,8 @@ do_test stmt-1.4 {
     INSERT INTO t1 SELECT a+1, b+1 FROM t1;
   }
   set sqlite_open_file_count
-} {3}
+  # 2016-03-04: statement-journal open deferred
+} {2}
 do_test stmt-1.5 {
   execsql COMMIT
   set sqlite_open_file_count
@@ -61,7 +62,8 @@ do_test stmt-1.6.1 {
 do_test stmt-1.6.2 {
   execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 }
   set sqlite_open_file_count
-} {3}
+  # 2016-03-04: statement-journal open deferred
+} {2}
 do_test stmt-1.7 {
   execsql COMMIT
   set sqlite_open_file_count
@@ -84,7 +86,7 @@ filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9   } 2
 filecount stmt-2.4 { 
     INSERT INTO t1 SELECT 9, 9;
     INSERT INTO t1 SELECT 10, 10;
-} 3
+} 2
 
 do_test stmt-2.5 {
   execsql { CREATE INDEX i1 ON t1(b) }
@@ -92,6 +94,6 @@ do_test stmt-2.5 {
 filecount stmt-2.6 { 
   REPLACE INTO t1 VALUES(5, 5);
   REPLACE INTO t1 VALUES(5, 5); 
-} 3
+} 2
 
 finish_test

test/tempdb.test

@@ -76,7 +76,7 @@ do_test tempdb-2.2 {
   }
   catchsql { INSERT INTO t1 SELECT * FROM t2 }
   set sqlite_open_file_count
-} [expr 1 + (0==$jrnl_in_memory) + (0==$subj_in_memory)]
+} [expr 1 + (0==$jrnl_in_memory)]
 do_test tempdb-2.3 {
   execsql {
     PRAGMA temp_store = 'memory';