From 2ca0f86354d2c87c02293cdd644d1ce53a8d2b7f Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Thu, 23 Aug 2007 08:06:44 +0000 Subject: [PATCH] Add some tests for the atomic-write optimization. (CVS 4275) FossilOrigin-Name: e2cc7b4a3476a733b2701546f6b4ec9abc18152b --- manifest | 20 ++--- manifest.uuid | 2 +- src/journal.c | 15 ++-- src/os.c | 36 +++++++-- src/test6.c | 200 +++++++++++++++++++++++++++++++------------------- test/io.test | 188 ++++++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 359 insertions(+), 102 deletions(-) diff --git a/manifest b/manifest index 0a8da1b3e0..0ac3a9a487 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\snames\sof\sconstants\sin\slemon.c\sto\swork\saround\sname\sconflicts\non\sSolaris.\s\sTicket\s#2583.\s(CVS\s4274) -D 2007-08-23T02:50:56 +C Add\ssome\stests\sfor\sthe\satomic-write\soptimization.\s(CVS\s4275) +D 2007-08-23T08:06:45 F Makefile.in 0c0e53720f658c7a551046442dd7afba0b72bfbe F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -94,7 +94,7 @@ F src/func.c aa8a3a8db571c46e5197664ddbc2784006ee5525 F src/hash.c 2f322979071dd2bdba7503b5276d66f028744382 F src/hash.h 3ad3da76bfb954978d227bf495568b0e6da2c19e F src/insert.c 633322aef1799f6604fa805e12488bc628570b0c -F src/journal.c 5ba2a1443b181741d3f2984d9d49e730073d74d1 +F src/journal.c 03d6b5cc1afe7c5e3cd0af55415f5168eb094398 F src/legacy.c 7e1b1c57694e49cbadf561e2a7d9cd984dc743b5 F src/limits.h 71ab25f17e35e0a9f3f6f234b8ed49cc56731d35 F src/loadext.c 8b31e2e0e961918fa045515459aee1c122d8c266 @@ -104,7 +104,7 @@ F src/md5.c c5fdfa5c2593eaee2e32a5ce6c6927c986eaf217 F src/mem1.c 7b023d45dd71944414db469c742457239e24d74d F src/mem2.c 48919353f72b8f6e957a0021eb9deaf863998189 F src/mutex.c 9cf641f556a4119ef90ed41b82f2d5647f81686e -F src/os.c d8f029317c95dcd2887b9f0f154281cdfbd303ad +F src/os.c 86593b6e8cc22304d7c2d24b06c0aae49254b181 F src/os.h 399c89cafa93b9ef35c3dc70f77644d10936b535 F src/os_common.h a5c446d3b93f09f369d13bf217de4bed3437dd1c F src/os_os2.c 8769301bff502de642ad2634cedcb77d967ce199 @@ -136,7 +136,7 @@ F src/test2.c 4f742e99ed1bea5c14692f627bdb59a146f30504 F src/test3.c a7d011c51d6b2e2a73c43983d5c2b731d69c74d7 F src/test4.c c2c0f5dc907f1346f5d4b65eb5799f11eb9e4071 F src/test5.c 3a6a5717a149d7ca2e6d14f5be72cf7555d54dc4 -F src/test6.c 1191c2305c85aa192f04186b59b05a4fb40018cc +F src/test6.c 2c141f367ba483eef99a7f4d00f07431caff791e F src/test7.c a9d509d0e9ad214b4772696f49f6e61be26213d1 F src/test8.c e6a543c8b248efe120ae33a6859fcd55dcf46a96 F src/test9.c b46c8fe02ac7cca1a7316436d8d38d50c66f4b2f @@ -318,7 +318,7 @@ F test/insert3.test 72ea6056811fd234f80d923f977c196089947381 F test/insert4.test 1e27f0a3e5670d5f03c1636f699aa44270945bca F test/interrupt.test 81555fb0f8179bb2d0dc7151fd75428223f93cf2 F test/intpkey.test af4fd826c4784ec5c93b444de07adea0254d0d30 -F test/io.test 6b7ee16f78560c4b81b52da2ea1051b8a2a93ce3 +F test/io.test ca9db7cd57a1b02cd983863c1be1153e1900e68b F test/ioerr.test 491d42c49bbec598966d26b01ed7901f55e5ee2d F test/ioerr2.test f938eadb12108048813869b86beee4a2f98e34b8 F test/join.test af0443185378b64878750aa1cf4b83c216f246b4 @@ -559,7 +559,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 21f6b31097692171c6493e6ca6de6acbd62dc595 -R eedd9c3af3e8b7da02c2c75af15e1919 -U drh -Z e69a56f3f634fc877bc9bea036855b24 +P e4e74cd0f9343448ea38e57f08bb4f0616825f31 +R 630334db112909a8743c928d15787556 +U danielk1977 +Z 01e41e90624e3bb8ea0ae5e4ad13b9f8 diff --git a/manifest.uuid b/manifest.uuid index 0ae2563d06..27c4775eb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4e74cd0f9343448ea38e57f08bb4f0616825f31 \ No newline at end of file +e2cc7b4a3476a733b2701546f6b4ec9abc18152b \ No newline at end of file diff --git a/src/journal.c b/src/journal.c index 21171cb6cb..dc4c6ee558 100644 --- a/src/journal.c +++ b/src/journal.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** -** @(#) $Id: journal.c,v 1.1 2007/08/22 11:22:04 danielk1977 Exp $ +** @(#) $Id: journal.c,v 1.2 2007/08/23 08:06:45 danielk1977 Exp $ */ #ifdef SQLITE_ENABLE_ATOMIC_WRITE @@ -54,11 +54,14 @@ typedef struct JournalFile JournalFile; static int createFile(JournalFile *p){ int rc = SQLITE_OK; if( !p->pReal ){ - p->pReal = (sqlite3_file *)&p[1]; - rc = sqlite3OsOpen(p->pVfs, p->zJournal, p->pReal, p->flags, 0); - if( rc==SQLITE_OK && p->iSize>0 ){ - assert(p->iSize<=p->nBuf); - rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); + sqlite3_file *pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); + if( rc==SQLITE_OK ){ + p->pReal = pReal; + if( p->iSize>0 ){ + assert(p->iSize<=p->nBuf); + rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); + } } } return rc; diff --git a/src/os.c b/src/os.c index 7eb6e973de..3ba530e77a 100644 --- a/src/os.c +++ b/src/os.c @@ -58,13 +58,35 @@ int sqlite3OsBreakLock(sqlite3_file *id){ int sqlite3OsCheckReservedLock(sqlite3_file *id){ return id->pMethods->xCheckReservedLock(id); } -int sqlite3OsSectorSize(sqlite3_file *id){ - int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; - return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); -} -int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ - return id->pMethods->xDeviceCharacteristics(id); -} + +#ifdef SQLITE_TEST + /* The following two variables are used to override the values returned + ** by the xSectorSize() and xDeviceCharacteristics() vfs methods for + ** testing purposes. They are usually set by a test command implemented + ** in test6.c. + */ + int sqlite3_test_sector_size = 0; + int sqlite3_test_device_characteristics = 0; + int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + int dc = id->pMethods->xDeviceCharacteristics(id); + return dc | sqlite3_test_device_characteristics; + } + int sqlite3OsSectorSize(sqlite3_file *id){ + if( sqlite3_test_sector_size==0 ){ + int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; + return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); + } + return sqlite3_test_sector_size; + } +#else + int sqlite3OsSectorSize(sqlite3_file *id){ + int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; + return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); + } + int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + return id->pMethods->xDeviceCharacteristics(id); + } +#endif #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) /* These methods are currently only used for testing and debugging. */ diff --git a/src/test6.c b/src/test6.c index 93befb8818..5a2ff13d4a 100644 --- a/src/test6.c +++ b/src/test6.c @@ -540,6 +540,98 @@ static int cfCurrentTime(void *pAppData, double *pTimeOut){ return pVfs->xCurrentTime(pVfs->pAppData, pTimeOut); } +static int processDevSymArgs( + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[], + int *piDeviceChar, + int *piSectorSize +){ + struct DeviceFlag { + char *zName; + int iValue; + } aFlag[] = { + { "atomic", SQLITE_IOCAP_ATOMIC }, + { "atomic512", SQLITE_IOCAP_ATOMIC512 }, + { "atomic1k", SQLITE_IOCAP_ATOMIC1K }, + { "atomic2k", SQLITE_IOCAP_ATOMIC2K }, + { "atomic4k", SQLITE_IOCAP_ATOMIC4K }, + { "atomic8k", SQLITE_IOCAP_ATOMIC8K }, + { "atomic16k", SQLITE_IOCAP_ATOMIC16K }, + { "atomic32k", SQLITE_IOCAP_ATOMIC32K }, + { "atomic64k", SQLITE_IOCAP_ATOMIC64K }, + { "sequential", SQLITE_IOCAP_SEQUENTIAL }, + { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, + { 0, 0 } + }; + + int i; + int iDc = 0; + int iSectorSize = 0; + int setSectorsize = 0; + int setDeviceChar = 0; + + for(i=0; i11 || nOpt<2 || strncmp("-sectorsize", zOpt, nOpt)) + && (nOpt>16 || nOpt<2 || strncmp("-characteristics", zOpt, nOpt)) + ){ + Tcl_AppendResult(interp, + "Bad option: \"", zOpt, + "\" - must be \"-characteristics\" or \"-sectorsize\"", 0 + ); + return TCL_ERROR; + } + if( i==objc-1 ){ + Tcl_AppendResult(interp, "Option requires an argument: \"", zOpt, "\"",0); + return TCL_ERROR; + } + + if( zOpt[1]=='s' ){ + if( Tcl_GetIntFromObj(interp, objv[i+1], &iSectorSize) ){ + return TCL_ERROR; + } + setSectorsize = 1; + }else{ + int j; + Tcl_Obj **apObj; + int nObj; + if( Tcl_ListObjGetElements(interp, objv[i+1], &nObj, &apObj) ){ + return TCL_ERROR; + } + for(j=0; j11 || nOpt<2 || strncmp("-sectorsize", zOpt, nOpt)) - && (nOpt>16 || nOpt<2 || strncmp("-characteristics", zOpt, nOpt)) - ){ - Tcl_AppendResult(interp, - "Bad option: \"", zOpt, - "\" - must be \"-characteristics\" or \"-sectorsize\"", 0 - ); - goto error; - } - if( i==objc-3 ){ - Tcl_AppendResult(interp, "Option requires an argument: \"", zOpt, "\"",0); - goto error; - } - - if( zOpt[1]=='s' ){ - if( Tcl_GetIntFromObj(interp, objv[i+1], &iSectorSize) ){ - goto error; - } - setSectorsize = 1; - }else{ - int j; - Tcl_Obj **apObj; - int nObj; - if( Tcl_ListObjGetElements(interp, objv[i+1], &nObj, &apObj) ){ - goto error; - } - for(j=0; j=0 ){ g.iDeviceCharacteristics = iDc; } - if( setSectorsize ){ + if( iSectorSize>=0 ){ g.iSectorSize = iSectorSize; } + g.iCrash = iDelay; memcpy(g.zCrashFile, zCrashFile, nCrashFile+1); sqlite3CrashTestEnable = 1; @@ -709,6 +734,32 @@ error: return TCL_ERROR; } +static int devSymObjCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + + extern int sqlite3_test_device_characteristics; + extern int sqlite3_test_sector_size; + + int iDc = -1; + int iSectorSize = -1; + if( processDevSymArgs(interp, objc-1, &objv[1], &iDc, &iSectorSize) ){ + return TCL_ERROR; + } + + if( iDc>=0 ){ + sqlite3_test_device_characteristics = iDc; + } + if( iSectorSize>=0 ){ + sqlite3_test_sector_size = iSectorSize; + } + + return TCL_OK; +} + #endif /* SQLITE_OMIT_DISKIO */ /* @@ -717,6 +768,7 @@ error: int Sqlitetest6_Init(Tcl_Interp *interp){ #ifndef SQLITE_OMIT_DISKIO Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0); + Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0); #endif return TCL_OK; } diff --git a/test/io.test b/test/io.test index 1cb264fff8..5ec9bcd9c4 100644 --- a/test/io.test +++ b/test/io.test @@ -13,11 +13,18 @@ # IO traffic generated by SQLite (making sure SQLite is not writing out # more database pages than it has to, stuff like that). # -# $Id: io.test,v 1.2 2007/08/22 02:56:44 drh Exp $ +# $Id: io.test,v 1.3 2007/08/23 08:06:45 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl +# Test summary: +# +# io-1.* - Test that quick-balance does not journal pages unnecessarily. +# io-2.* - Test that when the atomic-write optimisation is used no +# journal file is created. +# + set ::nWrite 0 proc nWrite {db} { set bt [btree_from_db $db] @@ -29,6 +36,13 @@ proc nWrite {db} { set res } +set ::nSync 0 +proc nSync {} { + set res [expr {$::sqlite_sync_count - $::nSync}] + set ::nSync $::sqlite_sync_count + set res +} + do_test io-1.1 { execsql { PRAGMA page_size = 1024; @@ -75,8 +89,6 @@ do_test io-1.4 { lappend ret [nWrite db] } {2 2 2} -#db eval {select * from sqlite_master} {btree_tree_dump [btree_from_db db] 2} - # This insert should use the quick-balance trick to add a third leaf # to the b-tree used to store table abc. It should only be necessary to # write to 3 pages to do this: the change-counter, the root-page and @@ -86,6 +98,174 @@ do_test io-1.5 { nWrite db } {3} -#db eval {select * from sqlite_master} {btree_tree_dump [btree_from_db db] 2} + + +#---------------------------------------------------------------------- +# Test cases io-2.* test the atomic-write optimization. +# +do_test io-2.1 { + execsql { DELETE FROM abc; VACUUM; } +} {} + +# Clear the write and sync counts. +nWrite db ; nSync + +# The following INSERT updates 2 pages and requires 4 calls to fsync(): +# +# 1) The directory in which the journal file is created, +# 2) The journal file (to sync the page data), +# 3) The journal file (to sync the journal file header), +# 4) The database file. +# +do_test io-2.2 { + execsql { INSERT INTO abc VALUES(1, 2) } + list [nWrite db] [nSync] +} {2 4} + +# Set the device-characteristic mask to include the SQLITE_IOCAP_ATOMIC, +# then do another INSERT similar to the one in io-2.2. This should +# only write 1 page and require a single fsync(). +# +# The single fsync() is the database file. Only one page is reported as +# written because page 1 - the change-counter page - is written using +# an out-of-band method that bypasses the write counter. +# +sqlite3_simulate_device -char atomic +do_test io-2.3 { + execsql { INSERT INTO abc VALUES(3, 4) } + list [nWrite db] [nSync] +} {1 1} + +# Test that the journal file is not created and the change-counter is +# updated when the atomic-write optimization is used. +# +do_test io-2.4.1 { + execsql { + BEGIN; + INSERT INTO abc VALUES(5, 6); + } + sqlite3 db2 test.db + execsql { SELECT * FROM abc } db2 +} {1 2 3 4} +do_test io-2.4.2 { + file exists test.db-journal +} {0} +do_test io-2.4.3 { + execsql { COMMIT } + execsql { SELECT * FROM abc } db2 +} {1 2 3 4 5 6} +db2 close + +# Test that the journal file is created and sync()d if the transaction +# modifies more than one database page, even if the IOCAP_ATOMIC flag +# is set. +# +do_test io-2.5.1 { + execsql { CREATE TABLE def(d, e) } + nWrite db ; nSync + execsql { + BEGIN; + INSERT INTO abc VALUES(7, 8); + } + file exists test.db-journal +} {0} +do_test io-2.5.2 { + execsql { INSERT INTO def VALUES('a', 'b'); } + file exists test.db-journal +} {1} +do_test io-2.5.3 { + execsql { COMMIT } + list [nWrite db] [nSync] +} {3 4} + +# Test that the journal file is created and sync()d if the transaction +# modifies a single database page and also appends a page to the file. +# Internally, this case is handled differently to the one above. The +# journal file is not actually created until the 'COMMIT' statement +# is executed. +# +do_test io-2.6.1 { + execsql { + BEGIN; + INSERT INTO abc VALUES(9, randstr(1000,1000)); + } + file exists test.db-journal +} {0} +do_test io-2.6.2 { + # Create a file at "test.db-journal". This will prevent SQLite from + # opening the journal for exclusive access. As a result, the COMMIT + # should fail with SQLITE_CANTOPEN and the transaction rolled back. + # + set fd [open test.db-journal w] + puts $fd "This is not a journal file" + close $fd + catchsql { COMMIT } +} {1 {unable to open database file}} +do_test io-2.6.3 { + file delete -force test.db-journal + catchsql { COMMIT } +} {1 {cannot commit - no transaction is active}} +do_test io-2.6.4 { + execsql { SELECT * FROM abc } +} {1 2 3 4 5 6 7 8} + + +# Test that if the database modification is part of multi-file commit, +# the journal file is always created. In this case, the journal file +# is created during execution of the COMMIT statement, so we have to +# use the same technique to check that it is created as in the above +# block. +file delete -force test2.db test2.db-journal +do_test io-2.7.1 { + execsql { + ATTACH 'test2.db' AS aux; + CREATE TABLE aux.abc2(a, b); + BEGIN; + INSERT INTO abc VALUES(9, 10); + } + file exists test.db-journal +} {0} +do_test io-2.7.2 { + execsql { INSERT INTO abc2 SELECT * FROM abc } + file exists test2.db-journal +} {0} +do_test io-2.7.3 { + execsql { SELECT * FROM abc UNION ALL SELECT * FROM abc2 } +} {1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10} +do_test io-2.7.4 { + set fd [open test2.db-journal w] + puts $fd "This is not a journal file" + close $fd + catchsql { COMMIT } +} {1 {unable to open database file}} +do_test io-2.7.5 { + file delete -force test2.db-journal + catchsql { COMMIT } +} {1 {cannot commit - no transaction is active}} +do_test io-2.7.6 { + execsql { SELECT * FROM abc UNION ALL SELECT * FROM abc2 } +} {1 2 3 4 5 6 7 8} + +# Try an explicit ROLLBACK before the journal file is created. +# +do_test io-2.8.1 { + execsql { + BEGIN; + DELETE FROM abc; + } + file exists test.db-journal +} {0} +do_test io-2.8.2 { + execsql { SELECT * FROM abc } +} {} +do_test io-2.8.3 { + execsql { + ROLLBACK; + SELECT * FROM abc; + } +} {1 2 3 4 5 6 7 8} + +sqlite3_simulate_device -char {} -sectorsize 0 finish_test +