diff --git a/manifest b/manifest index e74b367e97..cfd5eb0e64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\shyperlink\sin\sa\scomment. -D 2017-08-24T20:54:42.754 +C Update\sthe\smechanism\sused\sto\skeep\strack\sof\swhat\skind\sof\ssyncing\sto\sdo\sfor\nWAL\stransaction\scommits\sand\scheckpoint\soperations.\s\sUse\sthe\scheckpoint-style\nof\ssyncing\sto\ssync\sthe\sheader\sof\sa\snew\sor\srestarted\sWAL\sfile. +D 2017-08-25T01:14:43.498 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136 @@ -436,14 +436,14 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 +F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c c1dc0609f04a0659519bb2b8ca1440a64b0ad82b6c2afd675f1a50f6c918321a +F src/pager.c bf51378c57c8e05d7f4d7bb9861f403a2e40cde82e25513401216d1ed30bc3e5 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -535,8 +535,8 @@ F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 4c7c13a3c0d92b0bed157ac234679e4874afcb1990af33510542a0f35f9264b8 -F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 +F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a +F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c ad558533d6fe6578857635218633aa241e2428835763c46801be9e68d6ec0701 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d @@ -1486,7 +1486,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 56b0bc95b8693a0be294f8d210c49025dd094bd7 +F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1650,7 +1650,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe -R 0a9008b1e381e229108d828896139715 +P 25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a +R 0c45c0346ff1a3d7a9dbb07f83f4f723 +T *branch * wal-sync-refactor +T *sym-wal-sync-refactor * +T -sym-trunk * U drh -Z 0f9a691a1b4c4aa8a5eacf3d9e1dd571 +Z 5b1634e4e5e7d2e25acb68caf19e0213 diff --git a/manifest.uuid b/manifest.uuid index 404de58199..cfce901f7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a \ No newline at end of file +bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 5cf0014794..18277df356 100644 --- a/src/os.c +++ b/src/os.c @@ -98,7 +98,7 @@ int sqlite3OsTruncate(sqlite3_file *id, i64 size){ } int sqlite3OsSync(sqlite3_file *id, int flags){ DO_OS_MALLOC_TEST(id); - return id->pMethods->xSync(id, flags); + return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK; } int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ DO_OS_MALLOC_TEST(id); diff --git a/src/pager.c b/src/pager.c index 5986afdae1..2e5977fe43 100644 --- a/src/pager.c +++ b/src/pager.c @@ -616,6 +616,18 @@ struct PagerSavepoint { ** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode ** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. +** +** syncFlags, walSyncFlags +** +** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03). +** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode +** and contains the flags used to sync the checkpoint operations in the +** lower two bits, and sync flags used for transaction commits in the WAL +** file in bits 0x04 and 0x08. In other words, to get the correct sync flags +** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct +** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note +** that with synchronous=NORMAL in WAL mode, transaction commit is not synced +** meaning that the 0x04 and 0x08 bits are both zero. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -625,9 +637,8 @@ struct Pager { u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ u8 extraSync; /* sync directory after journal delete */ - u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ - u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 walSyncFlags; /* See description above */ u8 tempFile; /* zFilename is a temporary or immutable file */ u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ @@ -3598,20 +3609,17 @@ void sqlite3PagerSetFlags( } if( pPager->noSync ){ pPager->syncFlags = 0; - pPager->ckptSyncFlags = 0; }else if( pgFlags & PAGER_FULLFSYNC ){ pPager->syncFlags = SQLITE_SYNC_FULL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; - }else if( pgFlags & PAGER_CKPT_FULLFSYNC ){ - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; }else{ pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; } - pPager->walSyncFlags = pPager->syncFlags; + pPager->walSyncFlags = (pPager->syncFlags<<2); if( pPager->fullSync ){ - pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS; + pPager->walSyncFlags |= pPager->syncFlags; + } + if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){ + pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2); } if( pgFlags & PAGER_CACHESPILL ){ pPager->doNotSpill &= ~SPILLFLAG_OFF; @@ -4110,7 +4118,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL assert( db || pPager->pWal==0 ); - sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, pPager->pageSize, + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp) ); pPager->pWal = 0; @@ -4918,13 +4926,11 @@ act_like_temp_file: assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); - assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; pPager->extraSync = 0; pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); } /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ @@ -7372,7 +7378,7 @@ int sqlite3PagerCheckpoint( rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, - pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); } @@ -7529,7 +7535,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, + rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); diff --git a/src/wal.c b/src/wal.c index 45f1bc8c84..9930b84421 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1798,9 +1798,7 @@ static int walCheckpoint( pInfo->nBackfillAttempted = mxSafeFrame; /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); /* If the database may grow as a result of this checkpoint, hint ** about the eventual size of the db file to the VFS layer. @@ -1841,8 +1839,8 @@ static int walCheckpoint( i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); } } if( rc==SQLITE_OK ){ @@ -2948,8 +2946,8 @@ static int walWriteToLog( iOffset += iFirstAmt; iAmt -= iFirstAmt; pContent = (void*)(iFirstAmt + (char*)pContent); - assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) ); - rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK); + assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 ); + rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags)); if( iAmt==0 || rc ) return rc; } rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); @@ -3121,8 +3119,8 @@ int sqlite3WalFrames( ** ** https://sqlite.org/src/info/ff5be73dee */ - if( pWal->syncHeader && sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK); + if( pWal->syncHeader ){ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); if( rc ) return rc; } } @@ -3197,7 +3195,7 @@ int sqlite3WalFrames( ** sector boundary is synced; the part of the last frame that extends ** past the sector boundary is written after the sync. */ - if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ + if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){ int bSync = 1; if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); @@ -3213,7 +3211,7 @@ int sqlite3WalFrames( } if( bSync ){ assert( rc==SQLITE_OK ); - rc = sqlite3OsSync(w.pFd, sync_flags & SQLITE_SYNC_MASK); + rc = sqlite3OsSync(w.pFd, WAL_SYNC_FLAGS(sync_flags)); } } diff --git a/src/wal.h b/src/wal.h index 4f6d01dad6..d97300a684 100644 --- a/src/wal.h +++ b/src/wal.h @@ -19,11 +19,11 @@ #include "sqliteInt.h" -/* Additional values that can be added to the sync_flags argument of -** sqlite3WalFrames(): +/* Macros for extracting appropriate sync flags for either transaction +** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)): */ -#define WAL_SYNC_TRANSACTIONS 0x20 /* Sync at the end of each transaction */ -#define SQLITE_SYNC_MASK 0x13 /* Mask off the SQLITE_SYNC_* values */ +#define WAL_SYNC_FLAGS(X) ((X)&0x03) +#define CKPT_SYNC_FLAGS(X) (((X)>>2)&0x03) #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 diff --git a/test/wal2.test b/test/wal2.test index 0b15b15461..bc170ad3ff 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1191,7 +1191,7 @@ if {$::tcl_platform(platform) == "unix"} { # foreach {tn sql reslist} { 1 { } {10 0 4 0 6 0} - 2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2} + 2 { PRAGMA checkpoint_fullfsync = 1 } {10 6 4 3 6 3} 3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0} } { ifcapable default_ckptfullfsync { @@ -1261,8 +1261,8 @@ foreach {tn settings restart_sync commit_sync ckpt_sync} { 6 {0 1 full} {0 2} {0 1} {0 2} 7 {1 0 off} {0 0} {0 0} {0 0} - 8 {1 0 normal} {1 0} {0 0} {0 2} - 9 {1 0 full} {2 0} {1 0} {0 2} + 8 {1 0 normal} {0 1} {0 0} {0 2} + 9 {1 0 full} {1 1} {1 0} {0 2} 10 {1 1 off} {0 0} {0 0} {0 0} 11 {1 1 normal} {0 1} {0 0} {0 2}