From 3306c4a945de71d1313b2e090d1e3bd2a09821cd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Apr 2010 19:15:00 +0000 Subject: [PATCH] Fixes and tests for backup of a WAL database. FossilOrigin-Name: 480d12db4c0ebcc37598f7620d39193875eab15b --- manifest | 20 ++++----- manifest.uuid | 2 +- src/log.c | 8 ++++ src/log.h | 3 ++ src/pager.c | 55 ++++++++++++++++++++++-- test/backup.test | 3 ++ test/walbak.test | 110 ++++++++++++++++++++++++++++++++++++++++++++--- 7 files changed, 181 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 49b2e5bcc4..2f6f6317d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svery\ssimple\stest\scases\sfor\sbackup\sand\sVACUUM\sof\sWAL\sdatabases.\sMore\sto\scome. -D 2010-04-23T11:44:41 +C Fixes\sand\stests\sfor\sbackup\sof\sa\sWAL\sdatabase. +D 2010-04-23T19:15:00 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,8 +131,8 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6 -F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb +F src/log.c d9fdaad6b0b5ace54153b85f5bfcf1b9d1abac67 +F src/log.h bc44b0eec723648c8aa0a05ab78e1b76a4032e02 F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c d83d2ea6d2025554e079f891854b133924305e19 +F src/pager.c 485a34834a96863fd709f4d01f886ae70a9689ea F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -253,7 +253,7 @@ F test/autoindex1.test ffb06a246e2c1f89cfbe3d93eca513c9e78d4063 F test/autovacuum.test 25f891bc343a8bf5d9229e2e9ddab9f31a9ab5ec F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 F test/avtrans.test 1e901d8102706b63534dbd2bdd4d8f16c4082650 -F test/backup.test 3549ea8f541a08205c0eb813b21e81ea8301f6ed +F test/backup.test b1e874fa9b01de9dd5137a8371d060b76a435162 F test/backup2.test 159419073d9769fdb1780ed7e5b391a046f898d5 F test/backup_ioerr.test 1f012e692f42c0442ae652443258f70e9f20fa38 F test/backup_malloc.test 1e063c6d75143d0d6e0ae77971dd690070369387 @@ -759,7 +759,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49 -F test/walbak.test 3aecf824ee6433bd34673336623c4990aa1346ba +F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5d6d4423d1def39bd2424703120aa985085c3f8e -R 7813072bd3855cfe2a094b15eff01c69 +P 1077d8130b8ed5716ad73f78382270909d347963 +R 0d1581e70f8ee05579afbc11de72b135 U dan -Z f898bd1b5727c5103088c594c0575550 +Z c6fc40f98d5658443e299ebf167b2a61 diff --git a/manifest.uuid b/manifest.uuid index b5381319c7..4ef5c84110 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1077d8130b8ed5716ad73f78382270909d347963 \ No newline at end of file +480d12db4c0ebcc37598f7620d39193875eab15b \ No newline at end of file diff --git a/src/log.c b/src/log.c index f32fdb221d..c1b734f8fe 100644 --- a/src/log.c +++ b/src/log.c @@ -1568,6 +1568,14 @@ int sqlite3LogWriteLock(Log *pLog, int op){ return SQLITE_OK; } +/* +** Return true if data has been written but not committed to the log file. +*/ +int sqlite3LogDirty(Log *pLog){ + assert( pLog->isWriteLocked ); + return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg ); +} + /* ** Write a set of frames to the log. The caller must hold at least a ** RESERVED lock on the database file. diff --git a/src/log.h b/src/log.h index af698ebf5f..02201c837e 100644 --- a/src/log.h +++ b/src/log.h @@ -37,6 +37,9 @@ void sqlite3LogDbsize(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ int sqlite3LogWriteLock(Log *pLog, int op); +/* Return true if data has been written but not committed to the log file. */ +int sqlite3LogDirty(Log *pLog); + /* Write a frame or frames to the log. */ int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int); diff --git a/src/pager.c b/src/pager.c index 81f0968f54..2db8b9336c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2234,9 +2234,30 @@ static int readDbPage(PgHdr *pPg){ return rc; } +/* +** This function is called when a transaction on a WAL database is rolled +** back. For each dirty page in the cache, do one of the following: +** +** * If the page has no outstanding references, simply discard it. +** * Otherwise, if the page has one or more outstanding references, +** reload the original content from the database (or log file). +*/ static int pagerRollbackLog(Pager *pPager){ int rc = SQLITE_OK; PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + if( sqlite3LogDirty(pPager->pLog) ){ + sqlite3BackupRestart(pPager->pBackup); + } + pPager->dbSize = pPager->dbOrigSize; while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; @@ -3218,6 +3239,33 @@ static int subjournalPage(PgHdr *pPg){ return rc; } +/* +** This function is a wrapper around sqlite3LogFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. +*/ +static int pagerLogFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; /* Return code */ + + assert( pPager->pLog ); + rc = sqlite3LogFrames(pPager->pLog, + pPager->pageSize, pList, nTruncate, isCommit, sync_flags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + PgHdr *p; + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); + } + } + return rc; +} /* ** This function is called by the pcache layer when it has reached some @@ -3248,7 +3296,7 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseLog(pPager) ){ /* Write a single frame for this page to the log. */ - rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0); + rc = pagerLogFrames(pPager, pPg, 0, 0, 0); }else{ /* The doNotSync flag is set by the sqlite3PagerWrite() function while it ** is journalling a set of two or more database pages that are stored @@ -4881,8 +4929,8 @@ int sqlite3PagerCommitPhaseOne( if( pagerUseLog(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ - rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList, - pPager->dbSize, 1, (pPager->fullSync ? pPager->sync_flags : 0) + rc = pagerLogFrames(pPager, pList, pPager->dbSize, 1, + (pPager->fullSync ? pPager->sync_flags : 0) ); } sqlite3PcacheCleanAll(pPager->pPCache); @@ -5119,6 +5167,7 @@ int sqlite3PagerRollback(Pager *pPager){ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); if( pagerUseLog(pPager) ){ int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; diff --git a/test/backup.test b/test/backup.test index ec30adb361..ca8de8ecca 100644 --- a/test/backup.test +++ b/test/backup.test @@ -38,6 +38,8 @@ source $testdir/tester.tcl # # backup-9.*: Test that passing a negative argument to backup_step() is # interpreted as "copy the whole file". +# +# backup-10.*: Test writing the source database mid backup. # proc data_checksum {db file} { $db one "SELECT md5sum(a, b) FROM ${file}.t1" } @@ -905,6 +907,7 @@ ifcapable memorymanage { } +#----------------------------------------------------------------------- # Test that if the database is written to via the same database handle being # used as the source by a backup operation: # diff --git a/test/walbak.test b/test/walbak.test index f4998ac7f1..11a7040cdc 100644 --- a/test/walbak.test +++ b/test/walbak.test @@ -20,6 +20,12 @@ proc log_file_size {nFrame pgsz} { expr {12 + ($pgsz+16)*$nFrame} } +# Test organization: +# +# walback-1.*: Simple tests. +# walback-2.*: Test backups when the source db is modified mid-backup. +# + # Make sure a simple backup from a WAL database works. # do_test walbak-1.0 { @@ -53,20 +59,20 @@ db2 close # Try a VACUUM on a WAL database. # -do_test walbak-2.1 { +do_test walbak-1.4 { execsql { VACUUM; PRAGMA main.journal_mode; } } {wal} -do_test walbak-2.2 { +do_test walbak-1.5 { list [file size test.db] [file size test.db-wal] } [list 1024 [log_file_size 6 1024]] -do_test walbak-2.3 { +do_test walbak-1.6 { execsql { PRAGMA checkpoint } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 6 1024]] -do_test walbak-2.4 { +do_test walbak-1.7 { execsql { CREATE TABLE t2(a, b); INSERT INTO t2 SELECT * FROM t1; @@ -74,13 +80,105 @@ do_test walbak-2.4 { } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 6 1024]] -do_test walbak-2.5 { +do_test walbak-1.8 { execsql { VACUUM } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 8 1024]] -do_test walbak-2.6 { +do_test walbak-1.9 { execsql { PRAGMA checkpoint } list [file size test.db] [file size test.db-wal] } [list [expr 2*1024] [log_file_size 8 1024]] +#------------------------------------------------------------------------- +# Backups when the source db is modified mid-backup. +# +proc sig {{db db}} { + $db eval { + PRAGMA integrity_check; + SELECT md5sum(a, b) FROM t1; + } +} +db close +file delete test.db +sqlite3 db test.db +do_test walbak-2.1 { + execsql { PRAGMA journal_mode = WAL } + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + BEGIN; + INSERT INTO t1 VALUES(randomblob(500), randomblob(500)); + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 64 */ + COMMIT; + } +} {} +do_test walbak-2.2 { + db backup abc.db + sqlite3 db2 abc.db + string compare [sig db] [sig db2] +} {0} + +do_test walbak-2.3 { + sqlite3_backup B db2 main db main + B step 50 + execsql { UPDATE t1 SET b = randomblob(500) } + list [B step 1000] [B finish] +} {SQLITE_DONE SQLITE_OK} +do_test walbak-2.4 { + string compare [sig db] [sig db2] +} {0} + +do_test walbak-2.5 { + db close + sqlite3 db test.db + execsql { PRAGMA cache_size = 10 } + sqlite3_backup B db2 main db main + B step 50 + execsql { + BEGIN; + UPDATE t1 SET b = randomblob(500); + } + expr [file size test.db-wal] > 10*1024 +} {1} +do_test walbak-2.6 { + B step 1000 +} {SQLITE_BUSY} +do_test walbak-2.7 { + execsql COMMIT + list [B step 1000] [B finish] +} {SQLITE_DONE SQLITE_OK} +do_test walbak-2.8 { + string compare [sig db] [sig db2] +} {0} + +do_test walbak-2.9 { + db close + sqlite3 db test.db + execsql { PRAGMA cache_size = 10 } + sqlite3_backup B db2 main db main + B step 50 + execsql { + BEGIN; + UPDATE t1 SET b = randomblob(500); + } + expr [file size test.db-wal] > 10*1024 +} {1} +do_test walbak-2.10 { + B step 1000 +} {SQLITE_BUSY} +do_test walbak-2.11 { + execsql ROLLBACK +set sigB [sig db] + list [B step 1000] [B finish] +} {SQLITE_DONE SQLITE_OK} +do_test walbak-2.12 { + string compare [sig db] [sig db2] +} {0} +db2 close + finish_test +