From 94daf7fdffd34aa5eb0587954bb06def7941c72d Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Mon, 8 Nov 2004 09:26:09 +0000 Subject: [PATCH] Test auto-vacuum mode for crash-proofness. Also fix a bug related to the same. (CVS 2077) FossilOrigin-Name: 839ad771a6e781426c0fd624a4d1c91a5fcf8546 --- manifest | 18 ++++++------- manifest.uuid | 2 +- src/btree.c | 6 +---- src/pager.c | 28 +++++++++++++++++-- test/crash.test | 72 ++++++++++++++++++++++++++++++++++++++++++++++--- test/quick.test | 5 ++-- 6 files changed, 108 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 8790f59bdc..d5f60d0ade 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Auto-vacuum:\sAccount\sfor\sthe\spage\sreserved\sfor\swindows\slocking\s(PENDING_BYTE).\s(CVS\s2076) -D 2004-11-08T07:13:14 +C Test\sauto-vacuum\smode\sfor\scrash-proofness.\sAlso\sfix\sa\sbug\srelated\sto\sthe\ssame.\s(CVS\s2077) +D 2004-11-08T09:26:09 F Makefile.in c4d2416860f472a1e3393714d0372074197565df F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1 @@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea -F src/btree.c e88077ac041e153ba130488e5cef9475dd656966 +F src/btree.c 63a84350a18f6ca68f16e2a12018b5041444a2df F src/btree.h 861e40b759a195ba63819740e484390012cf81ab F src/build.c f01c2f9b3ad334a301e97ee4f299c36228ead110 F src/date.c 34bdb0082db7ec2a83ef00063f7b44e61ee19dad @@ -52,7 +52,7 @@ F src/os_unix.c 5824b22ba41fe9d514ef9169aac1b5fde73af229 F src/os_unix.h f3097815e041e82e24d92505e1ff61ba24172d13 F src/os_win.c 9482dfc92f289b68205bb2c9315757c7e3946bfb F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c 942cbb9d1de5e198d26979f992eba1b4e527ac08 +F src/pager.c 2653787b86267c079283b8128541095f0febac4f F src/pager.h 9eba8c53dd91eae7f3f90743b2ee242da02a9862 F src/parse.y 8456726833755ecd6dac9bcd8af205c8dc419d01 F src/pragma.c 6a0ae7721e614c5a921e918ab5206d5e654f1a6f @@ -109,7 +109,7 @@ F test/collate5.test 1dd5f0f508c46667f9d4606c7950c414b0bdc0d5 F test/collate6.test 6c9470d1606ee3e564675b229653e320c49ec638 F test/conflict.test c5b849b01cfbe0a4f63a90cba6f68e2fe3a75f87 F test/corrupt.test 0080ddcece23e8ba47c44608c4fb73fd4d1d8ce2 -F test/crash.test a3f6d27f7cb7f7bd752461db1e14f7c781ecedc3 +F test/crash.test 9982d2c1fe45126a96c57d16310caf949d21d818 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/date.test dda578ec1857837156bd8b32f8e09d81d7d7881c F test/delete.test fc29491f6a7ac899ce29f4549a104809e245d9a6 @@ -158,7 +158,7 @@ F test/pagesize.test 2050bb574c43f9143d283d6bcd37fa1eb73f0e49 F test/pragma.test bde1271384bc415af04d9dd736c073cf7ef33177 F test/printf.test 92ba4c510b4fc61120ffa4a01820446ed917ae57 F test/progress.test 5ddba78cb6011fba36093973cfb3ac473b8fb96a x -F test/quick.test 2dca186ebd5c418a7699944ba3b5e437d765eddd +F test/quick.test 9e968949a20b5ed5990c03dc45df3781a03c4b1a F test/quote.test 6d75cf635d93ba2484dc9cb378d88cbae9dc2c62 F test/reindex.test 3552c6b944a3fab28cfd3049c04c65cb79419757 F test/rollback.test 4097328d44510277244ef4fa51b22b2f11d7ef4c @@ -253,7 +253,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0 F www/whentouse.tcl fdacb0ba2d39831e8a6240d05a490026ad4c4e4c -P ad433ec2b6bd34e33dfe119668f38fbb978e889d -R f5ced09dda22d2b1aa5518cabc940350 +P d6335698696c7b651bbc436c5177d87eb57a8934 +R 631fbffc9d3d389ea9443eeeaba04e6a U danielk1977 -Z febc6549f1b02ac6b75a38defec39cf5 +Z 86fea8841f4dc937802f0e07bab9a3cf diff --git a/manifest.uuid b/manifest.uuid index a48996177e..800635e909 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6335698696c7b651bbc436c5177d87eb57a8934 \ No newline at end of file +839ad771a6e781426c0fd624a4d1c91a5fcf8546 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 748f6582de..d751f35ff4 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.214 2004/11/08 07:13:14 danielk1977 Exp $ +** $Id: btree.c,v 1.215 2004/11/08 09:26:09 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -1825,9 +1825,6 @@ static int autoVacuumCommit(Btree *pBt, Pgno *nTrunc){ *nTrunc = finSize; autovacuum_out: - /* TODO: A goto autovacuum_out; will fail to call releasePage() on - ** outstanding references. Fix. - */ assert( nRef==*sqlite3pager_stats(pPager) ); if( rc!=SQLITE_OK ){ sqlite3pager_rollback(pPager); @@ -5267,7 +5264,6 @@ char *sqlite3BtreeIntegrityCheck(Btree *pBt, int *aRoot, int nRoot){ for(i=0; iautoVacuum && aRoot[i]>1 ){ checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); } diff --git a/src/pager.c b/src/pager.c index b71ae59f18..43a31d8b64 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.175 2004/11/08 07:13:14 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.176 2004/11/08 09:26:10 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -34,7 +34,7 @@ #define TRACE2(X,Y) sqlite3DebugPrintf(X,Y) #define TRACE3(X,Y,Z) sqlite3DebugPrintf(X,Y,Z) #define TRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W) -#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W, V) +#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W,V) #else #define TRACE1(X) #define TRACE2(X,Y) @@ -3303,6 +3303,7 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){ PgHdr *pPg = DATA_TO_PGHDR(pData); PgHdr *pPgOld; int h; + Pgno needSyncPgno = 0; assert( !pPager->stmtInUse ); assert( pPg->nRef>0 ); @@ -3310,6 +3311,12 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){ TRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", PAGERID(pPager), pPg->pgno, pPg->needSync, pgno); + if( pPg->needSync ){ + needSyncPgno = pPg->pgno; + assert( pPg->inJournal ); + assert( pPg->dirty ); + } + /* Unlink pPg from it's hash-chain */ unlinkHashChain(pPager, pPg); @@ -3342,6 +3349,23 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){ pPg->dirty = 1; pPager->dirtyCache = 1; + if( needSyncPgno ){ + /* If needSyncPgno is non-zero, then the journal file needs to be + ** sync()ed before any data is written to database file page needSyncPgno. + ** Currently, no such page exists in the page-cache and the + ** Pager.aInJournal bit has been set. This needs to be remedied by loading + ** the page into the pager-cache and setting the PgHdr.needSync flag. + */ + int rc; + void *pNeedSync; + rc = sqlite3pager_get(pPager, needSyncPgno, &pNeedSync); + if( rc!=SQLITE_OK ) return rc; + DATA_TO_PGHDR(pNeedSync)->needSync = 1; + DATA_TO_PGHDR(pNeedSync)->inJournal = 1; + DATA_TO_PGHDR(pNeedSync)->dirty = 1; + sqlite3pager_unref(pNeedSync); + } + return SQLITE_OK; } #endif diff --git a/test/crash.test b/test/crash.test index 9bb4953fa3..d8a1836944 100644 --- a/test/crash.test +++ b/test/crash.test @@ -20,7 +20,7 @@ # The special crash-test module with its os_test.c backend only works # on Unix. # -# $Id: crash.test,v 1.9 2004/08/21 19:20:42 drh Exp $ +# $Id: crash.test,v 1.10 2004/11/08 09:26:10 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -68,6 +68,11 @@ proc signature2 {} { return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc2}] } +# This variable is set to 1 if the databases being used support auto-vacuum. +# This is because some of the tests in this file verify file-size, which is +# slightly larger for auto-vacuum databases. +set AUTOVACUUM [db eval {pragma auto_vacuum}] + #-------------------------------------------------------------------------- # Simple crash test: # @@ -170,7 +175,7 @@ do_test crash-2.1 { } {499500.0 999000.0 1498500.0} do_test crash-2.2 { expr [file size test.db] / 1024 -} {19} +} [expr $AUTOVACUUM ? 20 : 19] do_test crash-2.3 { crashsql 2 test.db-journal { DELETE FROM abc WHERE a < 800; @@ -210,7 +215,7 @@ do_test crash-3.0 { INSERT INTO abc SELECT * FROM abc; } expr [file size test.db] / 1024 -} {554} +} [expr $AUTOVACUUM ? 557 : 554] for {set i 1} {$i < $repeats} {incr i} { set sig [signature] do_test crash-3.$i.1 { @@ -324,3 +329,64 @@ for {set i 1} {$i < 5} {incr i} { signature2 } $sig2 } + +#-------------------------------------------------------------------------- +# The following test cases - crash-5.* - exposes a bug that existed in the +# sqlite3pager_movepage() API used by auto-vacuum databases. +# database when a crash occurs during a multi-file transaction. See comments +# in test crash-5.3 for details. +# +db close +file delete -force test.db +sqlite3 db test.db +do_test crash-5.1 { + execsql { + CREATE TABLE abc(a, b, c); -- Root page 3 + INSERT INTO abc VALUES(randstr(1500,1500), 0, 0); -- Overflow page 4 + INSERT INTO abc SELECT * FROM abc; + INSERT INTO abc SELECT * FROM abc; + INSERT INTO abc SELECT * FROM abc; + } +} {} +do_test crash-5.2 { + expr [file size test.db] / 1024 +} [expr $AUTOVACUUM ? 11 : 10] +set sig [signature] +do_test crash-5.3 { +# The SQL below is used to expose a bug that existed in +# sqlite3pager_movepage() during development of the auto-vacuum feature. It +# functions as follows: +# +# 1: Begin a transaction. +# 2: Put page 4 on the free-list (was the overflow page for the row deleted). +# 3: Write data to page 4 (it becomes the overflow page for the row inserted). +# The old page 4 data has been written to the journal file, but the +# journal file has not been sync()hronized. +# 4: Create a table, which calls sqlite3pager_movepage() to move page 4 +# to the end of the database (page 12) to make room for the new root-page. +# 5: Put pressure on the pager-cache. This results in page 4 being written +# to the database file to make space in the cache to load a new page. The +# bug was that page 4 was written to the database file before the journal +# is sync()hronized. +# 6: Commit. A crash occurs during the sync of the journal file. +# +# End result: Before the bug was fixed, data has been written to page 4 of the +# database file and the journal file does not contain trustworthy rollback +# data for this page. +# + crashsql 1 test.db-journal { + BEGIN; -- 1 + DELETE FROM abc WHERE oid = 1; -- 2 + INSERT INTO abc VALUES(randstr(1500,1500), 0, 0); -- 3 + CREATE TABLE abc2(a, b, c); -- 4 + SELECT * FROM abc; -- 5 + COMMIT; -- 6 + } +} {1 {child process exited abnormally}} +integrity_check crash-5.4 +do_test crash-5.5 { + signature +} $sig + + + diff --git a/test/quick.test b/test/quick.test index a8514dac9b..74c01c0dc4 100644 --- a/test/quick.test +++ b/test/quick.test @@ -10,7 +10,7 @@ #*********************************************************************** # This file runs all tests. # -# $Id: quick.test,v 1.31 2004/11/02 12:56:41 danielk1977 Exp $ +# $Id: quick.test,v 1.32 2004/11/08 09:26:10 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -32,8 +32,7 @@ set EXCLUDE { misuse.test quick.test utf16.test - - autovacuum.test + autovacuum_crash.test } if {[sqlite3 -has-codec]} {