Fix recent breakage of the appendvfs extension.

FossilOrigin-Name: 7dbbe5b34eea31ea54a5804035d4d0dfeb09c42cd9ab337e1e64c1296decbef3

ext/misc/appendvfs.c

@@ -38,8 +38,8 @@
 **  (5)  Otherwise, SQLITE_CANTOPEN is returned.
 **
 ** To avoid unnecessary complications with the PENDING_BYTE, the size of
-** the file containing the database is limited to 1GB. (1000013824 bytes)
+** the file containing the database is limited to 1GiB. (1073741824 bytes)
-** This VFS will not read or write past the 1GB mark.  This restriction
+** This VFS will not read or write past the 1GiB mark.  This restriction
 ** might be lifted in future versions.  For now, if you need a larger
 ** database, then keep it in a separate file.
 **
@@ -68,14 +68,16 @@ SQLITE_EXTENSION_INIT1
 ** Maximum size of the combined prefix + database + append-mark.  This
 ** must be less than 0x40000000 to avoid locking issues on Windows.
 */
-#define APND_MAX_SIZE  (65536*15259)
+#define APND_MAX_SIZE  (0x40000000)
 
 /*
-** Size of storage page upon which to align appendvfs portion.
+** Try to align the database to an even multiple of APND_ROUNDUP bytes.
 */
-#ifndef APND_ROUNDUP_BITS
+#ifndef APND_ROUNDUP
-#define APND_ROUNDUP_BITS 12
+#define APND_ROUNDUP 4096
 #endif
+#define APND_ALIGN_MASK         ((sqlite3_int64)(APND_ROUNDUP-1))
+#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK)
 
 /*
 ** Forward declaration of objects used by this utility
@@ -89,39 +91,45 @@ typedef struct ApndFile ApndFile;
 #define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
 #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
 
-/* Invariants for an open appendvfs file:
+/* An open appendvfs file
- * Once an appendvfs file is opened, it will be in one of three states:
+**
- * State 0: Never written. Underlying file (if any) is unaltered.
+** An instance of this structure describes the appended database file.
- * State 1: Append mark is persisted, content write is in progress.
+** A separate sqlite3_file object is always appended. The appended
- * State 2: Append mark is persisted, content writes are complete.
+** sqlite3_file object (which can be accessed using ORIGFILE()) describes
- * 
+** the entire file, including the prefix, the database, and the
- * State 0 is persistent in the sense that nothing will have been done
+** append-mark.
- * to the underlying file, including any attempt to convert it to an
+**
- * appendvfs file.
+** The structure of an AppendVFS database is like this:
- *
+**
- * State 1 is normally transitory. However, if a write operation ends
+**   +-------------+---------+----------+-------------+
- * abnormally (disk full, power loss, process kill, etc.), then State 1
+**   | prefix-file | padding | database | append-mark |
- * may be persistent on disk with an incomplete content write-out. This
+**   +-------------+---------+----------+-------------+
- * is logically equivalent to an interrupted write to an ordinary file,
+**                           ^          ^
- * where some unknown portion of to-be-written data is persisted while
+**                           |          |
- * the remainder is not. Database integrity in such cases is maintained
+**                         iPgOne      iMark
- * (or not) by the same measures available for ordinary file access.
+**
- *
+**
- * State 2 is persistent under normal circumstances (when there is no
+** "prefix file" -  file onto which the database has been appended.
- * abnormal termination of a write operation such that data provided
+** "padding"     -  zero or more bytes inserted so that "database"
- * to the underlying VFS write method has not yet reached storage.)
+**                  starts on an APND_ROUNDUP boundary
- *
+** "database"    -  The SQLite database file
- * In order to maintain the state invariant, the append mark is written
+** "append-mark" -  The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates
- * in advance of content writes where any part of such content would
+**                  the offset from the start of prefix-file to the start
- * overwrite an existing (or yet to be written) append mark.
+**                  of "database".
- */
+**
+** The size of the database is iMark - iPgOne.
+**
+** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value
+** of iPgOne stored as a big-ending 64-bit integer.
+**
+** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE).
+** Or, iMark is -1 to indicate that it has not yet been written.
+*/
 struct ApndFile {
-  /* Access to IO methods of the underlying file */
+  sqlite3_file base;        /* Subclass.  MUST BE FIRST! */
-  sqlite3_file base;
+  sqlite3_int64 iPgOne;     /* Offset to the start of the database */
-  /* File offset to beginning of appended content (unchanging) */
+  sqlite3_int64 iMark;      /* Offset of the append mark.  -1 if unwritten */
-  sqlite3_int64 iPgOne;
+  /* Always followed by another sqlite3_file that describes the whole file */
-  /* File offset of written append-mark, or -1 if unwritten */
-  sqlite3_int64 iMark;
 };
 
 /*
@@ -251,7 +259,7 @@ static int apndWriteMark(
   int rc;
   assert(pFile == ORIGFILE(paf));
   memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
-  while (--i >= 0) {
+  while( --i >= 0 ){
     a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff);
     iPgOne >>= 8;
   }
@@ -279,8 +287,7 @@ static int apndWrite(
   /* If append-mark is absent or will be overwritten, write it. */
   if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){
     int rc = apndWriteMark(paf, pFile, iWriteEnd);
-    if( SQLITE_OK!=rc )
+    if( SQLITE_OK!=rc ) return rc;
-      return rc;
   }
   return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst);
 }
@@ -292,8 +299,7 @@ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
   ApndFile *paf = (ApndFile *)pFile;
   pFile = ORIGFILE(pFile);
   /* The append mark goes out first so truncate failure does not lose it. */
-  if( SQLITE_OK!=apndWriteMark(paf, pFile, size) )
+  if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR;
-    return SQLITE_IOERR;
   /* Truncate underlying file just past append mark */
   return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE);
 }
@@ -409,8 +415,9 @@ static int apndFetch(
   void **pp
 ){
   ApndFile *p = (ApndFile *)pFile;
-  if( p->iMark < 0 || iOfst+iAmt > p->iMark)
+  if( p->iMark < 0 || iOfst+iAmt > p->iMark ){
     return SQLITE_IOERR; /* Cannot read what is not yet there. */
+  }
   pFile = ORIGFILE(pFile);
   return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
 }
@@ -461,14 +468,18 @@ static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){
   char zHdr[16];
   sqlite3_int64 iMark = apndReadMark(sz, pFile);
   if( iMark>=0 ){
-    /* If file has right end-marker, the expected odd size, and the
+    /* If file has the correct end-marker, the expected odd size, and the
-     * SQLite DB type marker where the end-marker puts it, then it
+    ** SQLite DB type marker where the end-marker puts it, then it
-     * is an appendvfs database (to be treated as such.)
+    ** is an appendvfs database.
-     */
+    */
     rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark);
-    if( SQLITE_OK==rc && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0
+    if( SQLITE_OK==rc
-        && (sz & 0x1ff)== APND_MARK_SIZE && sz>=512+APND_MARK_SIZE )
+     && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0
+     && (sz & 0x1ff) == APND_MARK_SIZE
+     && sz>=512+APND_MARK_SIZE
+    ){
       return 1; /* It's an appendvfs database */
+    }
   }
   return 0;
 }
@@ -490,66 +501,65 @@ static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
   }
 }
 
-/* Round-up used to get appendvfs portion to begin at a page boundary. */
-#define APND_ALIGN_MASK(nbits) ((1<<nbits)-1)
-#define APND_START_ROUNDUP(fsz, nbits) \
- ( ((fsz)+APND_ALIGN_MASK(nbits)) & ~(sqlite3_int64)APND_ALIGN_MASK(nbits) )
-
 /*
 ** Open an apnd file handle.
 */
 static int apndOpen(
-  sqlite3_vfs *pVfs,
+  sqlite3_vfs *pApndVfs,
   const char *zName,
   sqlite3_file *pFile,
   int flags,
   int *pOutFlags
 ){
-  ApndFile *p;
+  ApndFile *pApndFile = (ApndFile*)pFile;
-  sqlite3_file *pSubFile;
+  sqlite3_file *pBaseFile = ORIGFILE(pFile);
-  sqlite3_vfs *pSubVfs;
+  sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs);
   int rc;
   sqlite3_int64 sz;
-  pSubVfs = ORIGVFS(pVfs);
   if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
-    /* The appendvfs is not to be used for transient or temporary databases. */
+    /* The appendvfs is not to be used for transient or temporary databases.
-    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
+    ** Just use the base VFS open to initialize the given file object and
+    ** open the underlying file. (Appendvfs is then unused for this file.)
+    */
+    return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags);
   }
-  p = (ApndFile*)pFile;
+  memset(pApndFile, 0, sizeof(ApndFile));
-  memset(p, 0, sizeof(*p));
-  pSubFile = ORIGFILE(pFile);
   pFile->pMethods = &apnd_io_methods;
-  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
+  pApndFile->iMark = -1;    /* Append mark not yet written */
-  if( rc ) goto apnd_open_done;
+
-  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
+  rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags);
-  if( rc ){
+  if( rc==SQLITE_OK ){
-    pSubFile->pMethods->xClose(pSubFile);
+    rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz);
-    goto apnd_open_done;
   }
-  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
+  if( rc ){
-    memmove(pFile, pSubFile, pSubVfs->szOsFile);
+    pBaseFile->pMethods->xClose(pBaseFile);
+    pFile->pMethods = 0;
+    return rc;
+  }
+  if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){
+    /* The file being opened appears to be just an ordinary DB. Copy
+    ** the base dispatch-table so this instance mimics the base VFS. 
+    */
+    memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile);
     return SQLITE_OK;
   }
-  /* Record that append mark has not been written until seen otherwise. */
+  pApndFile->iPgOne = apndReadMark(sz, pFile);
-  p->iMark = -1;
+  if( pApndFile->iPgOne>=0 ){
-  p->iPgOne = apndReadMark(sz, pFile);
+    pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */
-  if( p->iPgOne>=0 ){
-    /* Append mark was found, infer its offset */
-    p->iMark = sz - p->iPgOne - APND_MARK_SIZE;
     return SQLITE_OK;
   }
   if( (flags & SQLITE_OPEN_CREATE)==0 ){
-    pSubFile->pMethods->xClose(pSubFile);
+    pBaseFile->pMethods->xClose(pBaseFile);
     rc = SQLITE_CANTOPEN;
+    pFile->pMethods = 0;
+  }else{
+    /* Round newly added appendvfs location to #define'd page boundary. 
+    ** Note that nothing has yet been written to the underlying file.
+    ** The append mark will be written along with first content write.
+    ** Until then, paf->iMark value indicates it is not yet written.
+    */
+    pApndFile->iPgOne = APND_START_ROUNDUP(sz);
   }
-  /* Round newly added appendvfs location to #define'd page boundary. 
-   * Note that nothing has yet been written to the underlying file.
-   * The append mark will be written along with first content write.
-   * Until then, the p->iMark value indicates it is not yet written.
-   */
-  p->iPgOne = APND_START_ROUNDUP(sz, APND_ROUNDUP_BITS);
-apnd_open_done:
-  if( rc ) pFile->pMethods = 0;
   return rc;
 }
 

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sALTER\sTABLE\sRENAME\sCOLUMN\swhen\sused\son\sa\sschema\sthat\sfeatures\sgenerated\scolumns.
+C Fix\srecent\sbreakage\sof\sthe\sappendvfs\sextension.
-D 2021-03-16T11:11:07.514
+D 2021-03-16T14:26:56.216
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240
 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c 7fff57cd4a5d63758d20a1dd1a96c64c7c123cf48fd98bbd36cfe8b063236e3d
+F ext/misc/appendvfs.c 390c411f14e66abb2f30b9023ebc57334288780c9388cdc6e0a25deb5d40e3b2
 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
 F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9
 F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c
@@ -699,7 +699,7 @@ F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf
 F test/autoindex5.test a5d72fe8c217cc0ea356dc6fa06a282a8a3fc53aa807709d79dba07a8f248102
 F test/autovacuum.test 0831cd34e14695d297187f7f6519265e3121c5b0a1720e548e86829e796129e9
 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
-F test/avfs.test 67a27e590b1baae5249bc857d10814a13231dbe2d624c64f2aa78c4ba85afff9
+F test/avfs.test 0c3a38e03cccb0fc3127838462dc05dc3f4c1480d770c084b388304c25de3652
 F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e
 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d
 F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989
@@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P aea12399bf1fdc76af43499d4624c3afa17c3e6c2459b71c195804bb98def66a
+P 0e255b26872b50581d470952dd98e21dd82d081885006f58d49daa4b4576b35d 25c3186aa143328157fb506a29802e87099cd14be4fa218230484a3212f20621
-R 8d9ba034916265e30952367e14ccf2d0
+R 7058d428dd88d4e4a708c5dd000f40bd
-U dan
+U drh
-Z 66abaaaa0bcfb13bbe669b759bf47eeb
+Z 11c2dd0109baa6fbadbea6ed933cddce

manifest.uuid

@@ -1 +1 @@
-0e255b26872b50581d470952dd98e21dd82d081885006f58d49daa4b4576b35d
+7dbbe5b34eea31ea54a5804035d4d0dfeb09c42cd9ab337e1e64c1296decbef3

test/avfs.test

@@ -20,6 +20,9 @@
 # avfs-2.1. Test that the simple text file retains its initial text.
 # avfs-3.1. Test that the appendvfs can grow and shrink, remaining intact.
 # avfs-3.2. Test that appendvfs is intact after grow/shrink/close/reopen.
+# avfs-3.3. Test that appendvfs can grow by many pages and be written.
+# avfs-3.4. Test that grown appendvfs can be reopened and appear intact.
+# avfs-3.5. Test that much grown appendvfs can shrink and reopen intact.
 # avfs-4.1. Test shell's ability to append to a non-appendvfs file.
 # avfs-4.2. Test shell's ability to append to empty or nonexistent file.
 # avfs-4.3. Test shell's ability to reopen and alter an appendvfs file.
@@ -159,11 +162,12 @@ do_test 2.1 {
 } {Appendee intact.}
 
 # Set of repeatable random integers for a couple tests.
+set ::nrint 50000
 proc rint {v} {
   return [::tcl::mathfunc::int [expr $v * 100000]]
 }
 array set ::randints [list 0 [rint [::tcl::mathfunc::srand 0]]]
-for {set i 1} {$i < 10000} {incr i} {
+for {set i 1} {$i < $::nrint} {incr i} {
   set ::randints($i) [rint [::tcl::mathfunc::rand]]
 }
 
@@ -176,7 +180,7 @@ do_test 3.1 {
     CREATE TABLE ri (i INTEGER);
     BEGIN;
   }
-  for {set i 0} {$i < 10000} {incr i} {
+  for {set i 0} {$i < $::nrint} {incr i} {
     set r $::randints($i)
     set s $::randints([incr i])
     set t $::randints([incr i])
@@ -203,11 +207,11 @@ do_test 3.1 {
   adb close
   set adaSz [file size $::fa]
   set adba [expr ($adbSz + 0.1)/$adaSz]
-  # lappend results $adbSz $adaSz
+  # lappend results $adba
   set results [concat $results [lrange $qr 0 2]]
-  lappend results [expr {$adba > 10.0 && $adba < 20.0}]
+  lappend results [expr {$adba > 10.0}]
   set ::result [join $results " | "]
-} {ok | 10000 | ok | ok | 1}
+} "ok | $::nrint | ok | ok | 1"
 
 do_test 3.2 {
   set results {}
@@ -219,6 +223,60 @@ do_test 3.2 {
   set ::result [join $results " | "]
 } {ok}
 
+# avfs-3.3. Test that appendvfs can grow by many pages and be written.
+do_test 3.3 {
+  set results {}
+  sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1
+  set npages 300
+  adb eval { BEGIN }
+  while {$npages > 0} {
+    adb eval { INSERT INTO ri VALUES (randomblob(1500)) }
+    incr npages -1
+  }
+  adb eval { COMMIT }
+  adb eval {
+    SELECT integrity_check as ic FROM pragma_integrity_check();
+  } { lappend results $ic }
+  adb close
+  set adaSzr [expr [file size $::fa] / 300.0 / 1500 ]
+  set okSzr [expr $adaSzr > 1.0 && $adaSzr < 1.3 ]
+  lappend results $okSzr
+  set ::result [join $results " | "]
+} {ok | 1}
+
+# avfs-3.4. Test that grown appendvfs can be reopened and appear intact.
+do_test 3.4 {
+  set results {}
+  sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1
+  adb eval {
+    SELECT integrity_check as ic FROM pragma_integrity_check();
+  } { lappend results $ic }
+  adb close
+  set ::result $ic
+} {ok}
+
+# avfs-3.5. Test that much grown appendvfs can shrink and reopen intact.
+do_test 3.5 {
+  set results {}
+  set adbsz [file size $::fa]
+  sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1
+  adb eval {
+    DELETE FROM ri WHERE rowid % 8 <> 0;
+    SELECT integrity_check as ic FROM pragma_integrity_check();
+    VACUUM;
+    SELECT integrity_check as ic FROM pragma_integrity_check();
+  } { lappend results $ic }
+  adb close
+  set adasz [file size $::fa]
+  lappend results [expr {$adbsz/$adasz > 5}]
+  sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1
+  adb eval {
+    SELECT integrity_check as ic FROM pragma_integrity_check();
+  } { lappend results $ic }
+  adb close
+  set ::result [join $results " | "]
+} {ok | ok | 1 | ok}
+
 set ::cliDoesAr [shellDoesAr]
 
 do_test 4.1 {
@@ -332,6 +390,6 @@ do_test 5.2 {
 
 forcedelete $::fa $::fza
 
-unset -nocomplain ::fa ::fza ::tlo ::result ::randints ::cliDoesAr
+unset -nocomplain ::fa ::fza ::tlo ::result ::randints ::nrint ::cliDoesAr
 
 finish_test