From 21f0f694ff32954a43a4518401ad3905bcbe32f9 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 16 Mar 2021 04:03:59 +0000 Subject: [PATCH 1/4] Create new branch named "appendvfs_fix" FossilOrigin-Name: 026edd601444d86858a503ffc3be17667a62a29f09c001009d9a678400b3b0a1 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index ea6ec02745..634081d3be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sgenerated\sautoconf/Makefile.msc\sto\sreflect\sits\ssource. -D 2021-03-12T23:15:08.537 +C Create\snew\sbranch\snamed\s"appendvfs_fix" +D 2021-03-16T04:03:59.658 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1910,7 +1910,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P acd63062eb06748bfe9e4886639e4f2b54ea6a496a83f10716abbaba4115500b +P d9f8f488ff9d47fe7bb8838e683bae4fea038f7278ef885ecf292143a0dd88ed R 17f134cb27374fc63f087829be8f2d98 +T *branch * appendvfs_fix +T *sym-appendvfs_fix * +T -sym-trunk * U larrybr -Z f38b52cd66603df216d7879edb7bd2e5 +Z bbee4d88175664ad835a258941f99787 diff --git a/manifest.uuid b/manifest.uuid index 214626662f..c53c0630b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9f8f488ff9d47fe7bb8838e683bae4fea038f7278ef885ecf292143a0dd88ed \ No newline at end of file +026edd601444d86858a503ffc3be17667a62a29f09c001009d9a678400b3b0a1 \ No newline at end of file From 5cad178b86f5ec6e8c29fff70b0392a274d81ebc Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 16 Mar 2021 06:41:51 +0000 Subject: [PATCH 2/4] Fix appendvfs bug exposed with bigger files, and add tests for such conditions. FossilOrigin-Name: 19b1f53a1c0a14440ae8ac71660a2595d37a4a5b201055c19366c7dca75d6660 --- ext/misc/appendvfs.c | 58 +++++++++++++++++++++--------------- manifest | 17 +++++------ manifest.uuid | 2 +- test/avfs.test | 70 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 107 insertions(+), 40 deletions(-) diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index 3fff9a10f6..970bb282c2 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -122,6 +122,10 @@ struct ApndFile { sqlite3_int64 iPgOne; /* File offset of written append-mark, or -1 if unwritten */ sqlite3_int64 iMark; + /* Whenever file is open and .iMark >= 0, and file size changes are + * not in progress, .iMark + sizeof(append mark) == base file size, + * and append file size == .iMark - .iPgOne . + */ }; /* @@ -217,6 +221,7 @@ static const sqlite3_io_methods apnd_io_methods = { ** Close an apnd-file. */ static int apndClose(sqlite3_file *pFile){ + assert((ApndFile *pFile)pFile == ORIGFILE(pFile)); pFile = ORIGFILE(pFile); return pFile->pMethods->xClose(pFile); } @@ -250,6 +255,7 @@ static int apndWriteMark( int i = APND_MARK_FOS_SZ; int rc; assert(pFile == ORIGFILE(paf)); + /* assert(pFile == paf->base); */ memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); while (--i >= 0) { a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); @@ -276,6 +282,7 @@ static int apndWrite( sqlite_int64 iWriteEnd = iOfst + iAmt; if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; pFile = ORIGFILE(pFile); + /* assert(pFile == paf->base); */ /* If append-mark is absent or will be overwritten, write it. */ if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ int rc = apndWriteMark(paf, pFile, iWriteEnd); @@ -499,55 +506,60 @@ static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ ** Open an apnd file handle. */ static int apndOpen( - sqlite3_vfs *pVfs, + sqlite3_vfs *pApndVfs, const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags ){ - ApndFile *p; - sqlite3_file *pSubFile; - sqlite3_vfs *pSubVfs; + ApndFile *pApndFile = (ApndFile*)pFile; + sqlite3_file *pBaseFile = ORIGFILE(pFile); + sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs); int rc; sqlite3_int64 sz; - pSubVfs = ORIGVFS(pVfs); if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ - /* The appendvfs is not to be used for transient or temporary databases. */ - return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + /* The appendvfs is not to be used for transient or temporary databases. + * Just use the base VFS open to initialize the given file object and + * open the underlying file. (Appendvfs is then unused for this file.) + */ + return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); } - p = (ApndFile*)pFile; - memset(p, 0, sizeof(*p)); - pSubFile = ORIGFILE(pFile); + memset(pApndFile, 0, sizeof(ApndFile)); pFile->pMethods = &apnd_io_methods; - rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + /* Record that append mark has not been written until seen otherwise. */ + pApndFile->iMark = -1; + + rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); if( rc ) goto apnd_open_done; - rc = pSubFile->pMethods->xFileSize(pSubFile, &sz); + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); if( rc ){ - pSubFile->pMethods->xClose(pSubFile); + pBaseFile->pMethods->xClose(pBaseFile); goto apnd_open_done; } - if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){ - memmove(pFile, pSubFile, pSubVfs->szOsFile); + if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ + /* The file being opened appears to be just an ordinary DB. Copy + * the base dispatch-table so this instance mimics the base VFS. + */ + memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); return SQLITE_OK; } - /* Record that append mark has not been written until seen otherwise. */ - p->iMark = -1; - p->iPgOne = apndReadMark(sz, pFile); - if( p->iPgOne>=0 ){ + pApndFile->iPgOne = apndReadMark(sz, pFile); + if( pApndFile->iPgOne>=0 ){ /* Append mark was found, infer its offset */ - p->iMark = sz - p->iPgOne - APND_MARK_SIZE; + pApndFile->iMark = sz - APND_MARK_SIZE; + /* pApndFile->base = pBaseFile; */ return SQLITE_OK; } if( (flags & SQLITE_OPEN_CREATE)==0 ){ - pSubFile->pMethods->xClose(pSubFile); + pBaseFile->pMethods->xClose(pBaseFile); rc = SQLITE_CANTOPEN; } /* Round newly added appendvfs location to #define'd page boundary. * Note that nothing has yet been written to the underlying file. * The append mark will be written along with first content write. - * Until then, the p->iMark value indicates it is not yet written. + * Until then, paf->iMark value indicates it is not yet written. */ - p->iPgOne = APND_START_ROUNDUP(sz, APND_ROUNDUP_BITS); + pApndFile->iPgOne = APND_START_ROUNDUP(sz, APND_ROUNDUP_BITS); apnd_open_done: if( rc ) pFile->pMethods = 0; return rc; diff --git a/manifest b/manifest index 634081d3be..1c494e6573 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\snew\sbranch\snamed\s"appendvfs_fix" -D 2021-03-16T04:03:59.658 +C Fix\sappendvfs\sbug\sexposed\swith\sbigger\sfiles,\sand\sadd\stests\sfor\ssuch\sconditions. +D 2021-03-16T06:41:51.332 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 7fff57cd4a5d63758d20a1dd1a96c64c7c123cf48fd98bbd36cfe8b063236e3d +F ext/misc/appendvfs.c 714118d14b610c00973aae124c1f8c24d338682bdef77bbe8f0cd2ff26952c65 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c @@ -699,7 +699,7 @@ F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf F test/autoindex5.test a5d72fe8c217cc0ea356dc6fa06a282a8a3fc53aa807709d79dba07a8f248102 F test/autovacuum.test 0831cd34e14695d297187f7f6519265e3121c5b0a1720e548e86829e796129e9 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 67a27e590b1baae5249bc857d10814a13231dbe2d624c64f2aa78c4ba85afff9 +F test/avfs.test 0c3a38e03cccb0fc3127838462dc05dc3f4c1480d770c084b388304c25de3652 F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 @@ -1910,10 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9f8f488ff9d47fe7bb8838e683bae4fea038f7278ef885ecf292143a0dd88ed -R 17f134cb27374fc63f087829be8f2d98 -T *branch * appendvfs_fix -T *sym-appendvfs_fix * -T -sym-trunk * +P 026edd601444d86858a503ffc3be17667a62a29f09c001009d9a678400b3b0a1 +R 3f7323614d0b0269053ff308d3cf0c77 U larrybr -Z bbee4d88175664ad835a258941f99787 +Z 1150e673d6f8e609bf947bc6b9f00cc8 diff --git a/manifest.uuid b/manifest.uuid index c53c0630b8..1e3cbd1570 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -026edd601444d86858a503ffc3be17667a62a29f09c001009d9a678400b3b0a1 \ No newline at end of file +19b1f53a1c0a14440ae8ac71660a2595d37a4a5b201055c19366c7dca75d6660 \ No newline at end of file diff --git a/test/avfs.test b/test/avfs.test index b2daa70a2d..2ebd608baa 100644 --- a/test/avfs.test +++ b/test/avfs.test @@ -20,6 +20,9 @@ # avfs-2.1. Test that the simple text file retains its initial text. # avfs-3.1. Test that the appendvfs can grow and shrink, remaining intact. # avfs-3.2. Test that appendvfs is intact after grow/shrink/close/reopen. +# avfs-3.3. Test that appendvfs can grow by many pages and be written. +# avfs-3.4. Test that grown appendvfs can be reopened and appear intact. +# avfs-3.5. Test that much grown appendvfs can shrink and reopen intact. # avfs-4.1. Test shell's ability to append to a non-appendvfs file. # avfs-4.2. Test shell's ability to append to empty or nonexistent file. # avfs-4.3. Test shell's ability to reopen and alter an appendvfs file. @@ -159,11 +162,12 @@ do_test 2.1 { } {Appendee intact.} # Set of repeatable random integers for a couple tests. +set ::nrint 50000 proc rint {v} { return [::tcl::mathfunc::int [expr $v * 100000]] } array set ::randints [list 0 [rint [::tcl::mathfunc::srand 0]]] -for {set i 1} {$i < 10000} {incr i} { +for {set i 1} {$i < $::nrint} {incr i} { set ::randints($i) [rint [::tcl::mathfunc::rand]] } @@ -176,7 +180,7 @@ do_test 3.1 { CREATE TABLE ri (i INTEGER); BEGIN; } - for {set i 0} {$i < 10000} {incr i} { + for {set i 0} {$i < $::nrint} {incr i} { set r $::randints($i) set s $::randints([incr i]) set t $::randints([incr i]) @@ -203,11 +207,11 @@ do_test 3.1 { adb close set adaSz [file size $::fa] set adba [expr ($adbSz + 0.1)/$adaSz] - # lappend results $adbSz $adaSz + # lappend results $adba set results [concat $results [lrange $qr 0 2]] - lappend results [expr {$adba > 10.0 && $adba < 20.0}] + lappend results [expr {$adba > 10.0}] set ::result [join $results " | "] -} {ok | 10000 | ok | ok | 1} +} "ok | $::nrint | ok | ok | 1" do_test 3.2 { set results {} @@ -219,6 +223,60 @@ do_test 3.2 { set ::result [join $results " | "] } {ok} +# avfs-3.3. Test that appendvfs can grow by many pages and be written. +do_test 3.3 { + set results {} + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 + set npages 300 + adb eval { BEGIN } + while {$npages > 0} { + adb eval { INSERT INTO ri VALUES (randomblob(1500)) } + incr npages -1 + } + adb eval { COMMIT } + adb eval { + SELECT integrity_check as ic FROM pragma_integrity_check(); + } { lappend results $ic } + adb close + set adaSzr [expr [file size $::fa] / 300.0 / 1500 ] + set okSzr [expr $adaSzr > 1.0 && $adaSzr < 1.3 ] + lappend results $okSzr + set ::result [join $results " | "] +} {ok | 1} + +# avfs-3.4. Test that grown appendvfs can be reopened and appear intact. +do_test 3.4 { + set results {} + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 + adb eval { + SELECT integrity_check as ic FROM pragma_integrity_check(); + } { lappend results $ic } + adb close + set ::result $ic +} {ok} + +# avfs-3.5. Test that much grown appendvfs can shrink and reopen intact. +do_test 3.5 { + set results {} + set adbsz [file size $::fa] + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 + adb eval { + DELETE FROM ri WHERE rowid % 8 <> 0; + SELECT integrity_check as ic FROM pragma_integrity_check(); + VACUUM; + SELECT integrity_check as ic FROM pragma_integrity_check(); + } { lappend results $ic } + adb close + set adasz [file size $::fa] + lappend results [expr {$adbsz/$adasz > 5}] + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 + adb eval { + SELECT integrity_check as ic FROM pragma_integrity_check(); + } { lappend results $ic } + adb close + set ::result [join $results " | "] +} {ok | ok | 1 | ok} + set ::cliDoesAr [shellDoesAr] do_test 4.1 { @@ -332,6 +390,6 @@ do_test 5.2 { forcedelete $::fa $::fza -unset -nocomplain ::fa ::fza ::tlo ::result ::randints ::cliDoesAr +unset -nocomplain ::fa ::fza ::tlo ::result ::randints ::nrint ::cliDoesAr finish_test From 67d4bbf6c1d75e56d66deb7f34d1521b49b307c1 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 16 Mar 2021 07:06:29 +0000 Subject: [PATCH 3/4] Fix assert typo. FossilOrigin-Name: 3aedf818c1cfc88ee1103abd8b20e787f6be1bcc4c9350963f0cd52f5ac04154 --- ext/misc/appendvfs.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index 970bb282c2..477e1eafc6 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -221,7 +221,7 @@ static const sqlite3_io_methods apnd_io_methods = { ** Close an apnd-file. */ static int apndClose(sqlite3_file *pFile){ - assert((ApndFile *pFile)pFile == ORIGFILE(pFile)); + assert((ApndFile*)pFile == ORIGFILE(pFile)); pFile = ORIGFILE(pFile); return pFile->pMethods->xClose(pFile); } diff --git a/manifest b/manifest index 00184c3928..8697813227 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfrom\strunk. -D 2021-03-16T06:50:39.815 +C Fix\sassert\stypo. +D 2021-03-16T07:06:29.878 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 714118d14b610c00973aae124c1f8c24d338682bdef77bbe8f0cd2ff26952c65 +F ext/misc/appendvfs.c 1dad6e7dc162ad36367513c1f985a1b474c3f3517f53ba22a6a89ce980843098 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 19b1f53a1c0a14440ae8ac71660a2595d37a4a5b201055c19366c7dca75d6660 aea12399bf1fdc76af43499d4624c3afa17c3e6c2459b71c195804bb98def66a -R 7ca2567aac2bb3bf35e5b9b280905d1f +P eae8236f3b9bc326648d1c8fca61cb3c38f821d06fe915fd0978cffeb1f6152b +R 32b396e0e85126b3386d72e2bc6e8e4c U larrybr -Z 4b3a7e0be93199f4ec9eb27099986229 +Z f8d4366505c35a746a63af41266ec4f5 diff --git a/manifest.uuid b/manifest.uuid index cd8ff74516..b9d2df6bbf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eae8236f3b9bc326648d1c8fca61cb3c38f821d06fe915fd0978cffeb1f6152b \ No newline at end of file +3aedf818c1cfc88ee1103abd8b20e787f6be1bcc4c9350963f0cd52f5ac04154 \ No newline at end of file From 3fee67533e265be377125faf3c73d26c90914f13 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Mar 2021 14:12:26 +0000 Subject: [PATCH 4/4] Comment and stylistic changes to the appendvfs.c implementation. FossilOrigin-Name: 25c3186aa143328157fb506a29802e87099cd14be4fa218230484a3212f20621 --- ext/misc/appendvfs.c | 164 +++++++++++++++++++++---------------------- manifest | 14 ++-- manifest.uuid | 2 +- 3 files changed, 89 insertions(+), 91 deletions(-) diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index 477e1eafc6..6975a6abff 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -38,8 +38,8 @@ ** (5) Otherwise, SQLITE_CANTOPEN is returned. ** ** To avoid unnecessary complications with the PENDING_BYTE, the size of -** the file containing the database is limited to 1GB. (1000013824 bytes) -** This VFS will not read or write past the 1GB mark. This restriction +** the file containing the database is limited to 1GiB. (1073741824 bytes) +** This VFS will not read or write past the 1GiB mark. This restriction ** might be lifted in future versions. For now, if you need a larger ** database, then keep it in a separate file. ** @@ -68,14 +68,16 @@ SQLITE_EXTENSION_INIT1 ** Maximum size of the combined prefix + database + append-mark. This ** must be less than 0x40000000 to avoid locking issues on Windows. */ -#define APND_MAX_SIZE (65536*15259) +#define APND_MAX_SIZE (0x40000000) /* -** Size of storage page upon which to align appendvfs portion. +** Try to align the database to an even multiple of APND_ROUNDUP bytes. */ -#ifndef APND_ROUNDUP_BITS -#define APND_ROUNDUP_BITS 12 +#ifndef APND_ROUNDUP +#define APND_ROUNDUP 4096 #endif +#define APND_ALIGN_MASK ((sqlite3_int64)(APND_ROUNDUP-1)) +#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK) /* ** Forward declaration of objects used by this utility @@ -89,43 +91,45 @@ typedef struct ApndFile ApndFile; #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) -/* Invariants for an open appendvfs file: - * Once an appendvfs file is opened, it will be in one of three states: - * State 0: Never written. Underlying file (if any) is unaltered. - * State 1: Append mark is persisted, content write is in progress. - * State 2: Append mark is persisted, content writes are complete. - * - * State 0 is persistent in the sense that nothing will have been done - * to the underlying file, including any attempt to convert it to an - * appendvfs file. - * - * State 1 is normally transitory. However, if a write operation ends - * abnormally (disk full, power loss, process kill, etc.), then State 1 - * may be persistent on disk with an incomplete content write-out. This - * is logically equivalent to an interrupted write to an ordinary file, - * where some unknown portion of to-be-written data is persisted while - * the remainder is not. Database integrity in such cases is maintained - * (or not) by the same measures available for ordinary file access. - * - * State 2 is persistent under normal circumstances (when there is no - * abnormal termination of a write operation such that data provided - * to the underlying VFS write method has not yet reached storage.) - * - * In order to maintain the state invariant, the append mark is written - * in advance of content writes where any part of such content would - * overwrite an existing (or yet to be written) append mark. - */ +/* An open appendvfs file +** +** An instance of this structure describes the appended database file. +** A separate sqlite3_file object is always appended. The appended +** sqlite3_file object (which can be accessed using ORIGFILE()) describes +** the entire file, including the prefix, the database, and the +** append-mark. +** +** The structure of an AppendVFS database is like this: +** +** +-------------+---------+----------+-------------+ +** | prefix-file | padding | database | append-mark | +** +-------------+---------+----------+-------------+ +** ^ ^ +** | | +** iPgOne iMark +** +** +** "prefix file" - file onto which the database has been appended. +** "padding" - zero or more bytes inserted so that "database" +** starts on an APND_ROUNDUP boundary +** "database" - The SQLite database file +** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates +** the offset from the start of prefix-file to the start +** of "database". +** +** The size of the database is iMark - iPgOne. +** +** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value +** of iPgOne stored as a big-ending 64-bit integer. +** +** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). +** Or, iMark is -1 to indicate that it has not yet been written. +*/ struct ApndFile { - /* Access to IO methods of the underlying file */ - sqlite3_file base; - /* File offset to beginning of appended content (unchanging) */ - sqlite3_int64 iPgOne; - /* File offset of written append-mark, or -1 if unwritten */ - sqlite3_int64 iMark; - /* Whenever file is open and .iMark >= 0, and file size changes are - * not in progress, .iMark + sizeof(append mark) == base file size, - * and append file size == .iMark - .iPgOne . - */ + sqlite3_file base; /* Subclass. MUST BE FIRST! */ + sqlite3_int64 iPgOne; /* Offset to the start of the database */ + sqlite3_int64 iMark; /* Offset of the append mark. -1 if unwritten */ + /* Always followed by another sqlite3_file that describes the whole file */ }; /* @@ -221,7 +225,6 @@ static const sqlite3_io_methods apnd_io_methods = { ** Close an apnd-file. */ static int apndClose(sqlite3_file *pFile){ - assert((ApndFile*)pFile == ORIGFILE(pFile)); pFile = ORIGFILE(pFile); return pFile->pMethods->xClose(pFile); } @@ -255,9 +258,8 @@ static int apndWriteMark( int i = APND_MARK_FOS_SZ; int rc; assert(pFile == ORIGFILE(paf)); - /* assert(pFile == paf->base); */ memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); - while (--i >= 0) { + while( --i >= 0 ){ a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); iPgOne >>= 8; } @@ -282,12 +284,10 @@ static int apndWrite( sqlite_int64 iWriteEnd = iOfst + iAmt; if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; pFile = ORIGFILE(pFile); - /* assert(pFile == paf->base); */ /* If append-mark is absent or will be overwritten, write it. */ if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ int rc = apndWriteMark(paf, pFile, iWriteEnd); - if( SQLITE_OK!=rc ) - return rc; + if( SQLITE_OK!=rc ) return rc; } return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); } @@ -299,8 +299,7 @@ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ ApndFile *paf = (ApndFile *)pFile; pFile = ORIGFILE(pFile); /* The append mark goes out first so truncate failure does not lose it. */ - if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) - return SQLITE_IOERR; + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR; /* Truncate underlying file just past append mark */ return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); } @@ -416,8 +415,9 @@ static int apndFetch( void **pp ){ ApndFile *p = (ApndFile *)pFile; - if( p->iMark < 0 || iOfst+iAmt > p->iMark) + if( p->iMark < 0 || iOfst+iAmt > p->iMark ){ return SQLITE_IOERR; /* Cannot read what is not yet there. */ + } pFile = ORIGFILE(pFile); return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); } @@ -468,14 +468,18 @@ static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ char zHdr[16]; sqlite3_int64 iMark = apndReadMark(sz, pFile); if( iMark>=0 ){ - /* If file has right end-marker, the expected odd size, and the - * SQLite DB type marker where the end-marker puts it, then it - * is an appendvfs database (to be treated as such.) - */ + /* If file has the correct end-marker, the expected odd size, and the + ** SQLite DB type marker where the end-marker puts it, then it + ** is an appendvfs database. + */ rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); - if( SQLITE_OK==rc && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 - && (sz & 0x1ff)== APND_MARK_SIZE && sz>=512+APND_MARK_SIZE ) + if( SQLITE_OK==rc + && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff) == APND_MARK_SIZE + && sz>=512+APND_MARK_SIZE + ){ return 1; /* It's an appendvfs database */ + } } return 0; } @@ -497,11 +501,6 @@ static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ } } -/* Round-up used to get appendvfs portion to begin at a page boundary. */ -#define APND_ALIGN_MASK(nbits) ((1<xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); } memset(pApndFile, 0, sizeof(ApndFile)); pFile->pMethods = &apnd_io_methods; - /* Record that append mark has not been written until seen otherwise. */ - pApndFile->iMark = -1; + pApndFile->iMark = -1; /* Append mark not yet written */ rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); - if( rc ) goto apnd_open_done; - rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); + if( rc==SQLITE_OK ){ + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); + } if( rc ){ pBaseFile->pMethods->xClose(pBaseFile); - goto apnd_open_done; + pFile->pMethods = 0; + return rc; } if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ /* The file being opened appears to be just an ordinary DB. Copy - * the base dispatch-table so this instance mimics the base VFS. - */ + ** the base dispatch-table so this instance mimics the base VFS. + */ memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); return SQLITE_OK; } pApndFile->iPgOne = apndReadMark(sz, pFile); if( pApndFile->iPgOne>=0 ){ - /* Append mark was found, infer its offset */ - pApndFile->iMark = sz - APND_MARK_SIZE; - /* pApndFile->base = pBaseFile; */ + pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */ return SQLITE_OK; } if( (flags & SQLITE_OPEN_CREATE)==0 ){ pBaseFile->pMethods->xClose(pBaseFile); rc = SQLITE_CANTOPEN; + pFile->pMethods = 0; + }else{ + /* Round newly added appendvfs location to #define'd page boundary. + ** Note that nothing has yet been written to the underlying file. + ** The append mark will be written along with first content write. + ** Until then, paf->iMark value indicates it is not yet written. + */ + pApndFile->iPgOne = APND_START_ROUNDUP(sz); } - /* Round newly added appendvfs location to #define'd page boundary. - * Note that nothing has yet been written to the underlying file. - * The append mark will be written along with first content write. - * Until then, paf->iMark value indicates it is not yet written. - */ - pApndFile->iPgOne = APND_START_ROUNDUP(sz, APND_ROUNDUP_BITS); -apnd_open_done: - if( rc ) pFile->pMethods = 0; return rc; } diff --git a/manifest b/manifest index 8697813227..57975f75a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sassert\stypo. -D 2021-03-16T07:06:29.878 +C Comment\sand\sstylistic\schanges\sto\sthe\sappendvfs.c\simplementation. +D 2021-03-16T14:12:26.803 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 1dad6e7dc162ad36367513c1f985a1b474c3f3517f53ba22a6a89ce980843098 +F ext/misc/appendvfs.c 390c411f14e66abb2f30b9023ebc57334288780c9388cdc6e0a25deb5d40e3b2 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eae8236f3b9bc326648d1c8fca61cb3c38f821d06fe915fd0978cffeb1f6152b -R 32b396e0e85126b3386d72e2bc6e8e4c -U larrybr -Z f8d4366505c35a746a63af41266ec4f5 +P 3aedf818c1cfc88ee1103abd8b20e787f6be1bcc4c9350963f0cd52f5ac04154 +R d4d0d407c45719772ccdc3772a06a769 +U drh +Z 2e9ac8f611b5ac99207230cf538d46c9 diff --git a/manifest.uuid b/manifest.uuid index b9d2df6bbf..283bb8a435 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aedf818c1cfc88ee1103abd8b20e787f6be1bcc4c9350963f0cd52f5ac04154 \ No newline at end of file +25c3186aa143328157fb506a29802e87099cd14be4fa218230484a3212f20621 \ No newline at end of file