From f26a1549ac363019e032f7d328d435e1c118b5fd Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Dec 2014 19:04:54 +0000 Subject: [PATCH] Add the SQLITE_CHECKPOINT_TRUNCATE option. FossilOrigin-Name: 8e20a43419e46b6b9d1f60ec7ea420bbfb3ef358 --- manifest | 29 +++++++++++--------- manifest.uuid | 2 +- src/main.c | 9 ++++--- src/pragma.c | 4 ++- src/sqlite.h.in | 21 +++++++++------ src/test1.c | 3 ++- src/vdbe.c | 1 + src/wal.c | 70 +++++++++++++++++++++++++++++++++++-------------- test/wal5.test | 43 +++++++++++++++++++++++++++++- 9 files changed, 133 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 7f9f9524d0..35fb1bb478 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\stwo\sunreachable\sbranches\sinto\sassert()\sstatements. -D 2014-12-02T16:16:47.791 +C Add\sthe\sSQLITE_CHECKPOINT_TRUNCATE\soption. +D 2014-12-02T19:04:54.595 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 54d0f4896cebc61ae5f831937464953780fe5346 +F src/main.c 34b895b5ebfc73cd690dcd9ac6d0eecb28ce729f F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -222,7 +222,7 @@ F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f +F src/pragma.c d54cdd40b63d608f2d95b7482c710690e3593a73 F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in c63db0117aeb749ca02b6016dbbbccbbbd9a141d +F src/sqlite.h.in 400bac7dd1294cb902b5eb85feed1689a2623ced F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c a0bce4f47da65b76c80e5f8bf9a5ef174603866a +F src/test1.c f5d7ecd3dd663b11f35269fd91f7090db0570903 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 +F src/vdbe.c 60217f3b8ab984b2e2bb9e3965276dd29e5efe5d F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 +F src/wal.c f09818db7ba6e31d7a681eb99f801a7722c731d9 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b @@ -1113,7 +1113,7 @@ F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b +F test/wal5.test 174cc1512e304a7dfa28ac30527e28ea02fc37df F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd @@ -1223,7 +1223,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d04f380e1bd17104b3cf76b64d0cfc79a726606 -R 22214da224af1bd07c2ba4ce527797c4 -U drh -Z 4469ca4ec037115204af98a9b9e41068 +P 61b31e771430f490fc2c4cef55046debc4a5f4f5 +R 837049ab8d5153c1422981d3f48166e4 +T *branch * checkpoint-truncate +T *sym-checkpoint-truncate * +T -sym-trunk * +U dan +Z ce82007701b4a39f21164e74e2380c4f diff --git a/manifest.uuid b/manifest.uuid index fe8f0fdad5..9aff9e9fbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61b31e771430f490fc2c4cef55046debc4a5f4f5 \ No newline at end of file +8e20a43419e46b6b9d1f60ec7ea420bbfb3ef358 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 1d34b13d6f..9ac3fde733 100644 --- a/src/main.c +++ b/src/main.c @@ -1936,10 +1936,11 @@ int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ return SQLITE_MISUSE; } diff --git a/src/pragma.c b/src/pragma.c index 543f265ba9..ab9a283629 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2195,7 +2195,7 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ @@ -2207,6 +2207,8 @@ void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } sqlite3VdbeSetNumCols(v, 3); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f1b917c308..807c9ee62c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7291,6 +7291,10 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, ** but not database readers. +** +**
SQLITE_CHECKPOINT_TRUNCATE
+** This mode works the same way as SQLITE_CHECKPOINT_RESTART except that, +** if successful, it also truncates the log file to zero bytes in size. ** ** ** If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -7306,11 +7310,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for +** The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. The busy-handler is also invoked while waiting for ** database readers as described above. If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as @@ -7349,9 +7353,10 @@ int sqlite3_wal_checkpoint_v2( ** documentation for additional information about the meaning and use of ** each of these values. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 +#define SQLITE_CHECKPOINT_FULL 1 +#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* ** CAPI3REF: Virtual Table Interface Configuration diff --git a/src/test1.c b/src/test1.c index 1c43861547..be4ad92c11 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5690,10 +5690,11 @@ static int test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", 0 }; + const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( objc!=3 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB MODE ?NAME?"); diff --git a/src/vdbe.c b/src/vdbe.c index 822bf80bb8..7f661b42b1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5724,6 +5724,7 @@ case OP_Checkpoint: { assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ diff --git a/src/wal.c b/src/wal.c index 3033444b41..849b776743 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1623,6 +1623,34 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +*/ +static void walRestartHdr(Wal *pWal){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + sqlite3_randomness(4, &aSalt[1]); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1775,19 +1803,34 @@ static int walCheckpoint( rc = SQLITE_OK; } - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ assert( mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* If this is a TRUNCATE checkpoint, also truncate the wal file + ** to zero bytes in size on disk. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } @@ -2573,7 +2616,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -2606,20 +2648,8 @@ static int walRestartLog(Wal *pWal){ ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; diff --git a/test/wal5.test b/test/wal5.test index 68750f1479..09c7d76040 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -55,7 +55,8 @@ foreach {testprefix do_wal_checkpoint} { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } - if {$a(-mode)!="restart" && $a(-mode)!="full"} { set a(-mode) passive } + set vals {restart full truncate} + if {[lsearch -exact $vals $a(-mode)]<0} { set a(-mode) passive } set cmd [list sqlite3_wal_checkpoint_v2 $dbhandle $a(-mode)] if {[info exists a(-db)]} { lappend sql $a(-db) } @@ -278,6 +279,11 @@ foreach {testprefix do_wal_checkpoint} { 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 + 11 TRUNCATE - {0 0 0} 3 + 12 TRUNCATE 1 {1 3 3} 1 + 13 TRUNCATE 2 {1 4 3} 2 + 14 TRUNCATE 3 {1 4 4} 3 + } { do_multiclient_test tn { setup_and_attach_aux @@ -348,6 +354,41 @@ foreach {testprefix do_wal_checkpoint} { do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} } + + # Test SQLITE_CHECKPOINT_TRUNCATE. + # + do_multiclient_test tn { + + code1 $do_wal_checkpoint + code2 $do_wal_checkpoint + code3 $do_wal_checkpoint + + do_test 3.$tn.1 { + sql1 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = normal; + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + file size test.db-wal + } [wal_file_size 8 1024] + + do_test 3.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} + do_test 3.$tn.3 { file size test.db-wal } 0 + + do_test 3.$tn.4 { + sql2 { SELECT * FROM t1 } + } {1 2 3 4} + + do_test 3.$tn.5 { + sql2 { INSERT INTO t1 VALUES('a', 'b') } + file size test.db-wal + } [wal_file_size 2 1024] + + } }