Add the ability to read from read-only WAL-mode database files as long as

the -wal and -shm files are present on disk.

FossilOrigin-Name: 00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3

manifest

@@ -1,5 +1,5 @@
-C Fix\sthe\sSQLITE_ENABLE_UPDATE_DELETE_LIMIT\sfunctionality\sso\sthat\sit\sworks\swith\sviews\sand\sWITHOUT\sROWID\stables.
+C Add\sthe\sability\sto\sread\sfrom\sread-only\sWAL-mode\sdatabase\sfiles\sas\slong\sas\nthe\s-wal\sand\s-shm\sfiles\sare\spresent\son\sdisk.
-D 2017-11-14T17:06:37.824
+D 2017-11-14T19:34:22.764
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -435,7 +435,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c
+F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -454,8 +454,8 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df
+F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
+F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -466,13 +466,13 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
-F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
+F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5b1e89ba279f4a4ab2f0975a7100d75be71e1a43a2df75a9c909d45bdd18c6ed
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
-F src/sqlite.h.in 6d96f09aac30a030c7674a47659d8156263d2ccad1aa5dae23a723f7166a0c37
+F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h abd4e64bc72906449d801d0e211265525239bc021bd9b7a72143c281fc24fa03
@@ -550,7 +550,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579
+F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db
@@ -1507,7 +1507,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef
+F test/wal2.test 2d81ffe2a02d9e5c7447b266f7153716cfcba7aecda5ed832db4544617399e29
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1532,7 +1532,9 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
-F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160
+F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
+F test/walro2.test 8812e514c968bf4ee317571fafedac43443360ae23edd7d0f4ef1eae0c13e8e8
+F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1675,7 +1677,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3711ef2366af8fefccaaa0a6ee520ce6bc9c74a4fe0666f0a85ef96be46e02d3 72be33f9c84de3ec4afc40549482417456ca82c1d16b473dc034b144055271e5
+P dae4a97a483bee1e6ac0271ddd28a0dffcebf7522edaf12eb5e0eba5fc62516a 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251
-R 4965ea97eeaee748543d2aea665dda83
+R 4dd08ae7170c8d270fe307a98bb5bfb2
-U dan
+T +closed 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251
-Z 07c3ca1d0e3a6855fa6ef907209d6a9f
+U drh
+Z 1fe131254421255cb83ffd7abf325441

manifest.uuid

@@ -1 +1 @@
-dae4a97a483bee1e6ac0271ddd28a0dffcebf7522edaf12eb5e0eba5fc62516a
+00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3

src/main.c

@@ -1314,7 +1314,7 @@ const char *sqlite3ErrName(int rc){
       case SQLITE_NOMEM:              zName = "SQLITE_NOMEM";             break;
       case SQLITE_READONLY:           zName = "SQLITE_READONLY";          break;
       case SQLITE_READONLY_RECOVERY:  zName = "SQLITE_READONLY_RECOVERY"; break;
-      case SQLITE_READONLY_CANTLOCK:  zName = "SQLITE_READONLY_CANTLOCK"; break;
+      case SQLITE_READONLY_CANTINIT:  zName = "SQLITE_READONLY_CANTINIT"; break;
       case SQLITE_READONLY_ROLLBACK:  zName = "SQLITE_READONLY_ROLLBACK"; break;
       case SQLITE_READONLY_DBMOVED:   zName = "SQLITE_READONLY_DBMOVED";  break;
       case SQLITE_INTERRUPT:          zName = "SQLITE_INTERRUPT";         break;

src/os_unix.c

@@ -4108,6 +4108,7 @@ struct unixShmNode {
   int szRegion;              /* Size of shared-memory regions */
   u16 nRegion;               /* Size of array apRegion */
   u8 isReadonly;             /* True if read-only */
+  u8 isUnlocked;             /* True if no DMS lock held */
   char **apRegion;           /* Array of mapped shared-memory regions */
   int nRef;                  /* Number of unixShm objects pointing to this */
   unixShm *pFirst;           /* All unixShm objects pointing to this */
@@ -4270,6 +4271,64 @@ static void unixShmPurge(unixFile *pFd){
   }
 }
 
+/*
+** The DMS lock has not yet been taken on shm file pShmNode. Attempt to
+** take it now. Return SQLITE_OK if successful, or an SQLite error
+** code otherwise.
+**
+** If the DMS cannot be locked because this is a readonly_shm=1 
+** connection and no other process already holds a lock, return
+** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1.
+*/
+static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
+  struct flock lock;
+  int rc = SQLITE_OK;
+
+  /* Use F_GETLK to determine the locks other processes are holding
+  ** on the DMS byte. If it indicates that another process is holding
+  ** a SHARED lock, then this process may also take a SHARED lock
+  ** and proceed with opening the *-shm file. 
+  **
+  ** Or, if no other process is holding any lock, then this process
+  ** is the first to open it. In this case take an EXCLUSIVE lock on the
+  ** DMS byte and truncate the *-shm file to zero bytes in size. Then
+  ** downgrade to a SHARED lock on the DMS byte.
+  **
+  ** If another process is holding an EXCLUSIVE lock on the DMS byte,
+  ** return SQLITE_BUSY to the caller (it will try again). An earlier
+  ** version of this code attempted the SHARED lock at this point. But
+  ** this introduced a subtle race condition: if the process holding
+  ** EXCLUSIVE failed just before truncating the *-shm file, then this
+  ** process might open and use the *-shm file without truncating it.
+  ** And if the *-shm file has been corrupted by a power failure or
+  ** system crash, the database itself may also become corrupt.  */
+  lock.l_whence = SEEK_SET;
+  lock.l_start = UNIX_SHM_DMS;
+  lock.l_len = 1;
+  lock.l_type = F_WRLCK;
+  if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
+    rc = SQLITE_IOERR_LOCK;
+  }else if( lock.l_type==F_UNLCK ){
+    if( pShmNode->isReadonly ){
+      pShmNode->isUnlocked = 1;
+      rc = SQLITE_READONLY_CANTINIT;
+    }else{
+      rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+      if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){
+        rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename);
+      }
+    }
+  }else if( lock.l_type==F_WRLCK ){
+    rc = SQLITE_BUSY;
+  }
+
+  if( rc==SQLITE_OK ){
+    assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK );
+    rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
+  }
+  return rc;
+}
+
 /*
 ** Open a shared-memory area associated with open database file pDbFd.  
 ** This particular implementation uses mmapped files.
@@ -4308,9 +4367,9 @@ static void unixShmPurge(unixFile *pFd){
 static int unixOpenSharedMemory(unixFile *pDbFd){
   struct unixShm *p = 0;          /* The connection to be opened */
   struct unixShmNode *pShmNode;   /* The underlying mmapped file */
-  int rc;                         /* Result code */
+  int rc = SQLITE_OK;             /* Result code */
   unixInodeInfo *pInode;          /* The inode of fd */
-  char *zShmFilename;             /* Name of the file used for SHM */
+  char *zShm;             /* Name of the file used for SHM */
   int nShmFilename;               /* Size of the SHM filename in bytes */
 
   /* Allocate space for the new unixShm object. */
@@ -4351,14 +4410,14 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       goto shm_open_err;
     }
     memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename);
-    zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1];
+    zShm = pShmNode->zFilename = (char*)&pShmNode[1];
 #ifdef SQLITE_SHM_DIRECTORY
-    sqlite3_snprintf(nShmFilename, zShmFilename, 
+    sqlite3_snprintf(nShmFilename, zShm, 
                      SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x",
                      (u32)sStat.st_ino, (u32)sStat.st_dev);
 #else
-    sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath);
+    sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath);
-    sqlite3FileSuffix3(pDbFd->zPath, zShmFilename);
+    sqlite3FileSuffix3(pDbFd->zPath, zShm);
 #endif
     pShmNode->h = -1;
     pDbFd->pInode->pShmNode = pShmNode;
@@ -4372,15 +4431,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
     }
 
     if( pInode->bProcessLock==0 ){
-      int openFlags = O_RDWR | O_CREAT;
+      if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
-      if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
+        pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777));
-        openFlags = O_RDONLY;
-        pShmNode->isReadonly = 1;
       }
-      pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777));
       if( pShmNode->h<0 ){
-        rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename);
+        pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777));
-        goto shm_open_err;
+        if( pShmNode->h<0 ){
+          rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm);
+          goto shm_open_err;
+        }
+        pShmNode->isReadonly = 1;
       }
 
       /* If this process is running as root, make sure that the SHM file
@@ -4388,20 +4448,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       ** the original owner will not be able to connect.
       */
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
-  
+
-      /* Check to see if another process is holding the dead-man switch.
+      rc = unixLockSharedMemory(pDbFd, pShmNode);
-      ** If not, truncate the file to zero length. 
+      if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;
-      */
-      rc = SQLITE_OK;
-      if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){
-        if( robust_ftruncate(pShmNode->h, 0) ){
-          rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
-        }
-      }
-      if( rc==SQLITE_OK ){
-        rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
-      }
-      if( rc ) goto shm_open_err;
     }
   }
 
@@ -4425,7 +4474,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
   p->pNext = pShmNode->pFirst;
   pShmNode->pFirst = p;
   sqlite3_mutex_leave(pShmNode->mutex);
-  return SQLITE_OK;
+  return rc;
 
   /* Jump here on any error */
 shm_open_err:
@@ -4477,6 +4526,11 @@ static int unixShmMap(
   p = pDbFd->pShm;
   pShmNode = p->pShmNode;
   sqlite3_mutex_enter(pShmNode->mutex);
+  if( pShmNode->isUnlocked ){
+    rc = unixLockSharedMemory(pDbFd, pShmNode);
+    if( rc!=SQLITE_OK ) goto shmpage_out;
+    pShmNode->isUnlocked = 0;
+  }
   assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
   assert( pShmNode->pInode==pDbFd->pInode );
   assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 );

src/os_win.c

@@ -3673,6 +3673,9 @@ struct winShmNode {
 
   int szRegion;              /* Size of shared-memory regions */
   int nRegion;               /* Size of array apRegion */
+  u8 isReadonly;             /* True if read-only */
+  u8 isUnlocked;             /* True if no DMS lock held */
+
   struct ShmRegion {
     HANDLE hMap;             /* File handle from CreateFileMapping */
     void *pMap;
@@ -3820,6 +3823,37 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
   }
 }
 
+/*
+** The DMS lock has not yet been taken on shm file pShmNode. Attempt to
+** take it now. Return SQLITE_OK if successful, or an SQLite error
+** code otherwise.
+**
+** If the DMS cannot be locked because this is a readonly_shm=1
+** connection and no other process already holds a lock, return
+** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1.
+*/
+static int winLockSharedMemory(winShmNode *pShmNode){
+  int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1);
+
+  if( rc==SQLITE_OK ){
+    if( pShmNode->isReadonly ){
+      pShmNode->isUnlocked = 1;
+      winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+      return SQLITE_READONLY_CANTINIT;
+    }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){
+      winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+      return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(),
+                         "winLockSharedMemory", pShmNode->zFilename);
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+  }
+
+  return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
+}
+
 /*
 ** Open the shared-memory area associated with database file pDbFd.
 **
@@ -3829,9 +3863,10 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
 */
 static int winOpenSharedMemory(winFile *pDbFd){
   struct winShm *p;                  /* The connection to be opened */
-  struct winShmNode *pShmNode = 0;   /* The underlying mmapped file */
+  winShmNode *pShmNode = 0;          /* The underlying mmapped file */
-  int rc;                            /* Result code */
+  int rc = SQLITE_OK;                /* Result code */
-  struct winShmNode *pNew;           /* Newly allocated winShmNode */
+  int rc2 = SQLITE_ERROR;            /* winOpen result code */
+  winShmNode *pNew;                  /* Newly allocated winShmNode */
   int nName;                         /* Size of zName in bytes */
 
   assert( pDbFd->pShm==0 );    /* Not previously opened */
@@ -3878,30 +3913,29 @@ static int winOpenSharedMemory(winFile *pDbFd){
       }
     }
 
-    rc = winOpen(pDbFd->pVfs,
+    if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
-                 pShmNode->zFilename,             /* Name of the file (UTF-8) */
+      rc2 = winOpen(pDbFd->pVfs,
-                 (sqlite3_file*)&pShmNode->hFile,  /* File handle here */
+                    pShmNode->zFilename,
-                 SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
+                    (sqlite3_file*)&pShmNode->hFile,
-                 0);
+                    SQLITE_OPEN_WAL|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,
-    if( SQLITE_OK!=rc ){
+                    0);
-      goto shm_open_err;
+    }
+    if( rc2!=SQLITE_OK ){
+      rc2 = winOpen(pDbFd->pVfs,
+                    pShmNode->zFilename,
+                    (sqlite3_file*)&pShmNode->hFile,
+                    SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY,
+                    0);
+      if( rc2!=SQLITE_OK ){
+        rc = winLogError(rc2, osGetLastError(), "winOpenShm",
+                         pShmNode->zFilename);
+        goto shm_open_err;
+      }
+      pShmNode->isReadonly = 1;
     }
 
-    /* Check to see if another process is holding the dead-man switch.
+    rc = winLockSharedMemory(pShmNode);
-    ** If not, truncate the file to zero length.
+    if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;
-    */
-    if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){
-      rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0);
-      if( rc!=SQLITE_OK ){
-        rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(),
-                         "winOpenShm", pDbFd->zPath);
-      }
-    }
-    if( rc==SQLITE_OK ){
-      winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
-      rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
-    }
-    if( rc ) goto shm_open_err;
   }
 
   /* Make the new connection a child of the winShmNode */
@@ -3924,7 +3958,7 @@ static int winOpenSharedMemory(winFile *pDbFd){
   p->pNext = pShmNode->pFirst;
   pShmNode->pFirst = p;
   sqlite3_mutex_leave(pShmNode->mutex);
-  return SQLITE_OK;
+  return rc;
 
   /* Jump here on any error */
 shm_open_err:
@@ -4128,6 +4162,8 @@ static int winShmMap(
   winFile *pDbFd = (winFile*)fd;
   winShm *pShm = pDbFd->pShm;
   winShmNode *pShmNode;
+  DWORD protect = PAGE_READWRITE;
+  DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ;
   int rc = SQLITE_OK;
 
   if( !pShm ){
@@ -4138,6 +4174,11 @@ static int winShmMap(
   pShmNode = pShm->pShmNode;
 
   sqlite3_mutex_enter(pShmNode->mutex);
+  if( pShmNode->isUnlocked ){
+    rc = winLockSharedMemory(pShmNode);
+    if( rc!=SQLITE_OK ) goto shmpage_out;
+    pShmNode->isUnlocked = 0;
+  }
   assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
 
   if( pShmNode->nRegion<=iRegion ){
@@ -4184,21 +4225,26 @@ static int winShmMap(
     }
     pShmNode->aRegion = apNew;
 
+    if( pShmNode->isReadonly ){
+      protect = PAGE_READONLY;
+      flags = FILE_MAP_READ;
+    }
+
     while( pShmNode->nRegion<=iRegion ){
       HANDLE hMap = NULL;         /* file-mapping handle */
       void *pMap = 0;             /* Mapped memory region */
 
 #if SQLITE_OS_WINRT
       hMap = osCreateFileMappingFromApp(pShmNode->hFile.h,
-          NULL, PAGE_READWRITE, nByte, NULL
+          NULL, protect, nByte, NULL
       );
 #elif defined(SQLITE_WIN32_HAS_WIDE)
       hMap = osCreateFileMappingW(pShmNode->hFile.h,
-          NULL, PAGE_READWRITE, 0, nByte, NULL
+          NULL, protect, 0, nByte, NULL
       );
 #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA
       hMap = osCreateFileMappingA(pShmNode->hFile.h,
-          NULL, PAGE_READWRITE, 0, nByte, NULL
+          NULL, protect, 0, nByte, NULL
       );
 #endif
       OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n",
@@ -4208,11 +4254,11 @@ static int winShmMap(
         int iOffset = pShmNode->nRegion*szRegion;
         int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity;
 #if SQLITE_OS_WINRT
-        pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ,
+        pMap = osMapViewOfFileFromApp(hMap, flags,
             iOffset - iOffsetShift, szRegion + iOffsetShift
         );
 #else
-        pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ,
+        pMap = osMapViewOfFile(hMap, flags,
             0, iOffset - iOffsetShift, szRegion + iOffsetShift
         );
 #endif
@@ -4243,6 +4289,7 @@ shmpage_out:
   }else{
     *pp = 0;
   }
+  if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY;
   sqlite3_mutex_leave(pShmNode->mutex);
   return rc;
 }

src/printf.c

@@ -1092,8 +1092,15 @@ void sqlite3DebugPrintf(const char *zFormat, ...){
   sqlite3VXPrintf(&acc, zFormat, ap);
   va_end(ap);
   sqlite3StrAccumFinish(&acc);
+#ifdef SQLITE_OS_TRACE_PROC
+  {
+    extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf);
+    SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf));
+  }
+#else
   fprintf(stdout,"%s", zBuf);
   fflush(stdout);
+#endif
 }
 #endif
 

src/sqlite.h.in

@@ -508,11 +508,13 @@ int sqlite3_exec(
 #define SQLITE_CANTOPEN_ISDIR          (SQLITE_CANTOPEN | (2<<8))
 #define SQLITE_CANTOPEN_FULLPATH       (SQLITE_CANTOPEN | (3<<8))
 #define SQLITE_CANTOPEN_CONVPATH       (SQLITE_CANTOPEN | (4<<8))
+#define SQLITE_CANTOPEN_DIRTYWAL       (SQLITE_CANTOPEN | (5<<8))
 #define SQLITE_CORRUPT_VTAB            (SQLITE_CORRUPT | (1<<8))
 #define SQLITE_READONLY_RECOVERY       (SQLITE_READONLY | (1<<8))
 #define SQLITE_READONLY_CANTLOCK       (SQLITE_READONLY | (2<<8))
 #define SQLITE_READONLY_ROLLBACK       (SQLITE_READONLY | (3<<8))
 #define SQLITE_READONLY_DBMOVED        (SQLITE_READONLY | (4<<8))
+#define SQLITE_READONLY_CANTINIT       (SQLITE_READONLY | (5<<8))
 #define SQLITE_ABORT_ROLLBACK          (SQLITE_ABORT | (2<<8))
 #define SQLITE_CONSTRAINT_CHECK        (SQLITE_CONSTRAINT | (1<<8))
 #define SQLITE_CONSTRAINT_COMMITHOOK   (SQLITE_CONSTRAINT | (2<<8))

src/wal.c

@@ -455,6 +455,7 @@ struct Wal {
   u8 truncateOnCommit;       /* True to truncate WAL file on commit */
   u8 syncHeader;             /* Fsync the WAL header if true */
   u8 padToSectorBoundary;    /* Pad transactions out to the next sector */
+  u8 bShmUnreliable;         /* SHM content is read-only and unreliable */
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
@@ -544,6 +545,11 @@ struct WalIterator {
 ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are
 ** numbered from zero.
 **
+** If the wal-index is currently smaller the iPage pages then the size
+** of the wal-index might be increased, but only if it is safe to do
+** so.  It is safe to enlarge the wal-index if pWal->writeLock is true
+** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE.
+**
 ** If this call is successful, *ppPage is set to point to the wal-index
 ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs,
 ** then an SQLite error code is returned and *ppPage is set to 0.
@@ -575,9 +581,13 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
       rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
           pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
       );
-      if( rc==SQLITE_READONLY ){
+      assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 );
+      testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK );
+      if( (rc&0xff)==SQLITE_READONLY ){
         pWal->readOnly |= WAL_SHM_RDONLY;
-        rc = SQLITE_OK;
+        if( rc==SQLITE_READONLY ){
+          rc = SQLITE_OK;
+        }
       }
     }
   }
@@ -1099,7 +1109,6 @@ static int walIndexRecover(Wal *pWal){
   i64 nSize;                      /* Size of log file */
   u32 aFrameCksum[2] = {0, 0};
   int iLock;                      /* Lock offset to lock for checkpoint */
-  int nLock;                      /* Number of locks to hold */
 
   /* Obtain an exclusive lock on all byte in the locking range not already
   ** locked by the caller. The caller is guaranteed to have locked the
@@ -1112,11 +1121,17 @@ static int walIndexRecover(Wal *pWal){
   assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE );
   assert( pWal->writeLock );
   iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock;
-  nLock = SQLITE_SHM_NLOCK - iLock;
+  rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
-  rc = walLockExclusive(pWal, iLock, nLock);
+  if( rc==SQLITE_OK ){
+    rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+    if( rc!=SQLITE_OK ){
+      walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+    }
+  }
   if( rc ){
     return rc;
   }
+
   WALTRACE(("WAL%p: recovery begin...\n", pWal));
 
   memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
@@ -1254,7 +1269,8 @@ finished:
 
 recovery_error:
   WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok"));
-  walUnlockExclusive(pWal, iLock, nLock);
+  walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
   return rc;
 }
 
@@ -1262,13 +1278,14 @@ recovery_error:
 ** Close an open wal-index.
 */
 static void walIndexClose(Wal *pWal, int isDelete){
-  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){
     int i;
     for(i=0; i<pWal->nWiData; i++){
       sqlite3_free((void *)pWal->apWiData[i]);
       pWal->apWiData[i] = 0;
     }
-  }else{
+  }
+  if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){
     sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
   }
 }
@@ -2061,6 +2078,12 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){
   return 0;
 }
 
+/*
+** This is the value that walTryBeginRead returns when it needs to
+** be retried.
+*/
+#define WAL_RETRY  (-1)
+
 /*
 ** Read the wal-index header from the wal-index and into pWal->hdr.
 ** If the wal-header appears to be corrupt, try to reconstruct the
@@ -2084,9 +2107,29 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   assert( pChanged );
   rc = walIndexPage(pWal, 0, &page0);
   if( rc!=SQLITE_OK ){
-    return rc;
+    assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */
-  };
+    if( rc==SQLITE_READONLY_CANTINIT ){
-  assert( page0 || pWal->writeLock==0 );
+      /* The SQLITE_READONLY_CANTINIT return means that the shared-memory
+      ** was openable but is not writable, and this thread is unable to
+      ** confirm that another write-capable connection has the shared-memory
+      ** open, and hence the content of the shared-memory is unreliable,
+      ** since the shared-memory might be inconsistent with the WAL file
+      ** and there is no writer on hand to fix it. */
+      assert( page0==0 );
+      assert( pWal->writeLock==0 );
+      assert( pWal->readOnly & WAL_SHM_RDONLY );
+      pWal->bShmUnreliable = 1;
+      pWal->exclusiveMode = WAL_HEAPMEMORY_MODE;
+      *pChanged = 1;
+    }else{
+      return rc; /* Any other non-OK return is just an error */
+    }
+  }else{
+    /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock
+    ** is zero, which prevents the SHM from growing */
+    testcase( page0!=0 );
+  }
+  assert( page0!=0 || pWal->writeLock==0 );
 
   /* If the first page of the wal-index has been mapped, try to read the
   ** wal-index header immediately, without holding any lock. This usually
@@ -2100,7 +2143,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   */
   assert( badHdr==0 || pWal->writeLock==0 );
   if( badHdr ){
-    if( pWal->readOnly & WAL_SHM_RDONLY ){
+    if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){
       if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){
         walUnlockShared(pWal, WAL_WRITE_LOCK);
         rc = SQLITE_READONLY_RECOVERY;
@@ -2130,15 +2173,193 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){
     rc = SQLITE_CANTOPEN_BKPT;
   }
+  if( pWal->bShmUnreliable ){
+    if( rc!=SQLITE_OK ){
+      walIndexClose(pWal, 0);
+      pWal->bShmUnreliable = 0;
+      assert( pWal->nWiData>0 && pWal->apWiData[0]==0 );
+      /* walIndexRecover() might have returned SHORT_READ if a concurrent
+      ** writer truncated the WAL out from under it.  If that happens, it
+      ** indicates that a writer has fixed the SHM file for us, so retry */
+      if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY;
+    }
+    pWal->exclusiveMode = WAL_NORMAL_MODE;
+  }
 
   return rc;
 }
 
 /*
-** This is the value that walTryBeginRead returns when it needs to
+** Open a transaction in a connection where the shared-memory is read-only
-** be retried.
+** and where we cannot verify that there is a separate write-capable connection
+** on hand to keep the shared-memory up-to-date with the WAL file.
+**
+** This can happen, for example, when the shared-memory is implemented by
+** memory-mapping a *-shm file, where a prior writer has shut down and
+** left the *-shm file on disk, and now the present connection is trying
+** to use that database but lacks write permission on the *-shm file.
+** Other scenarios are also possible, depending on the VFS implementation.
+**
+** Precondition:
+**
+**    The *-wal file has been read and an appropriate wal-index has been
+**    constructed in pWal->apWiData[] using heap memory instead of shared
+**    memory. 
+**
+** If this function returns SQLITE_OK, then the read transaction has
+** been successfully opened. In this case output variable (*pChanged) 
+** is set to true before returning if the caller should discard the
+** contents of the page cache before proceeding. Or, if it returns 
+** WAL_RETRY, then the heap memory wal-index has been discarded and 
+** the caller should retry opening the read transaction from the 
+** beginning (including attempting to map the *-shm file). 
+**
+** If an error occurs, an SQLite error code is returned.
 */
-#define WAL_RETRY  (-1)
+static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
+  i64 szWal;                      /* Size of wal file on disk in bytes */
+  i64 iOffset;                    /* Current offset when reading wal file */
+  u8 aBuf[WAL_HDRSIZE];           /* Buffer to load WAL header into */
+  u8 *aFrame = 0;                 /* Malloc'd buffer to load entire frame */
+  int szFrame;                    /* Number of bytes in buffer aFrame[] */
+  u8 *aData;                      /* Pointer to data part of aFrame buffer */
+  volatile void *pDummy;          /* Dummy argument for xShmMap */
+  int rc;                         /* Return code */
+  u32 aSaveCksum[2];              /* Saved copy of pWal->hdr.aFrameCksum */
+
+  assert( pWal->bShmUnreliable );
+  assert( pWal->readOnly & WAL_SHM_RDONLY );
+  assert( pWal->nWiData>0 && pWal->apWiData[0] );
+
+  /* Take WAL_READ_LOCK(0). This has the effect of preventing any
+  ** writers from running a checkpoint, but does not stop them
+  ** from running recovery.  */
+  rc = walLockShared(pWal, WAL_READ_LOCK(0));
+  if( rc!=SQLITE_OK ){
+    if( rc==SQLITE_BUSY ) rc = WAL_RETRY;
+    goto begin_unreliable_shm_out;
+  }
+  pWal->readLock = 0;
+
+  /* Check to see if a separate writer has attached to the shared-memory area,
+  ** thus making the shared-memory "reliable" again.  Do this by invoking
+  ** the xShmMap() routine of the VFS and looking to see if the return
+  ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT.
+  **
+  ** If the shared-memory is now "reliable" return WAL_RETRY, which will
+  ** cause the heap-memory WAL-index to be discarded and the actual
+  ** shared memory to be used in its place.
+  **
+  ** This step is important because, even though this connection is holding
+  ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might
+  ** have already checkpointed the WAL file and, while the current
+  ** is active, wrap the WAL and start overwriting frames that this
+  ** process wants to use.
+  **
+  ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has
+  ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY
+  ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations,
+  ** even if some external agent does a "chmod" to make the shared-memory
+  ** writable by us, until sqlite3OsShmUnmap() has been called.
+  ** This is a requirement on the VFS implementation.
+   */
+  rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
+  assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */
+  if( rc!=SQLITE_READONLY_CANTINIT ){
+    rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc);
+    goto begin_unreliable_shm_out;
+  }
+
+  /* We reach this point only if the real shared-memory is still unreliable.
+  ** Assume the in-memory WAL-index substitute is correct and load it
+  ** into pWal->hdr.
+  */
+  memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
+
+  /* Make sure some writer hasn't come in and changed the WAL file out
+  ** from under us, then disconnected, while we were not looking.
+  */
+  rc = sqlite3OsFileSize(pWal->pWalFd, &szWal);
+  if( rc!=SQLITE_OK ){
+    goto begin_unreliable_shm_out;
+  }
+  if( szWal<WAL_HDRSIZE ){
+    /* If the wal file is too small to contain a wal-header and the
+    ** wal-index header has mxFrame==0, then it must be safe to proceed
+    ** reading the database file only. However, the page cache cannot
+    ** be trusted, as a read/write connection may have connected, written
+    ** the db, run a checkpoint, truncated the wal file and disconnected
+    ** since this client's last read transaction.  */
+    *pChanged = 1;
+    rc = (pWal->hdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY);
+    goto begin_unreliable_shm_out;
+  }
+
+  /* Check the salt keys at the start of the wal file still match. */
+  rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
+  if( rc!=SQLITE_OK ){
+    goto begin_unreliable_shm_out;
+  }
+  if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){
+    /* Some writer has wrapped the WAL file while we were not looking.
+    ** Return WAL_RETRY which will cause the in-memory WAL-index to be
+    ** rebuilt. */
+    rc = WAL_RETRY;
+    goto begin_unreliable_shm_out;
+  }
+
+  /* Allocate a buffer to read frames into */
+  szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE;
+  aFrame = (u8 *)sqlite3_malloc64(szFrame);
+  if( aFrame==0 ){
+    rc = SQLITE_NOMEM_BKPT;
+    goto begin_unreliable_shm_out;
+  }
+  aData = &aFrame[WAL_FRAME_HDRSIZE];
+
+  /* Check to see if a complete transaction has been appended to the
+  ** wal file since the heap-memory wal-index was created. If so, the
+  ** heap-memory wal-index is discarded and WAL_RETRY returned to
+  ** the caller.  */
+  aSaveCksum[0] = pWal->hdr.aFrameCksum[0];
+  aSaveCksum[1] = pWal->hdr.aFrameCksum[1];
+  for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); 
+      iOffset+szFrame<=szWal; 
+      iOffset+=szFrame
+  ){
+    u32 pgno;                   /* Database page number for frame */
+    u32 nTruncate;              /* dbsize field from frame header */
+
+    /* Read and decode the next log frame. */
+    rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset);
+    if( rc!=SQLITE_OK ) break;
+    if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break;
+
+    /* If nTruncate is non-zero, then a complete transaction has been
+    ** appended to this wal file. Set rc to WAL_RETRY and break out of
+    ** the loop.  */
+    if( nTruncate ){
+      rc = WAL_RETRY;
+      break;
+    }
+  }
+  pWal->hdr.aFrameCksum[0] = aSaveCksum[0];
+  pWal->hdr.aFrameCksum[1] = aSaveCksum[1];
+
+ begin_unreliable_shm_out:
+  sqlite3_free(aFrame);
+  if( rc!=SQLITE_OK ){
+    int i;
+    for(i=0; i<pWal->nWiData; i++){
+      sqlite3_free((void*)pWal->apWiData[i]);
+      pWal->apWiData[i] = 0;
+    }
+    pWal->bShmUnreliable = 0;
+    sqlite3WalEndReadTransaction(pWal);
+    *pChanged = 1;
+  }
+  return rc;
+}
 
 /*
 ** Attempt to start a read transaction.  This might fail due to a race or
@@ -2200,6 +2421,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
 
   assert( pWal->readLock<0 );     /* Not currently locked */
 
+  /* useWal may only be set for read/write connections */
+  assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 );
+
   /* Take steps to avoid spinning forever if there is a protocol error.
   **
   ** Circumstances that cause a RETRY should only last for the briefest
@@ -2228,7 +2452,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   }
 
   if( !useWal ){
-    rc = walIndexReadHdr(pWal, pChanged);
+    assert( rc==SQLITE_OK );
+    if( pWal->bShmUnreliable==0 ){
+      rc = walIndexReadHdr(pWal, pChanged);
+    }
     if( rc==SQLITE_BUSY ){
       /* If there is not a recovery running in another thread or process
       ** then convert BUSY errors to WAL_RETRY.  If recovery is known to
@@ -2257,10 +2484,15 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     if( rc!=SQLITE_OK ){
       return rc;
     }
+    else if( pWal->bShmUnreliable ){
+      return walBeginShmUnreliable(pWal, pChanged);
+    }
   }
 
+  assert( pWal->nWiData>0 );
+  assert( pWal->apWiData[0]!=0 );
   pInfo = walCkptInfo(pWal);
-  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame 
+  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame
 #ifdef SQLITE_ENABLE_SNAPSHOT
    && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
      || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
@@ -2334,7 +2566,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   }
   if( mxI==0 ){
     assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
-    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK;
+    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
   }
 
   rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
@@ -2606,7 +2838,7 @@ int sqlite3WalFindFrame(
   ** then the WAL is ignored by the reader so return early, as if the 
   ** WAL were empty.
   */
-  if( iLast==0 || pWal->readLock==0 ){
+  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
     *piRead = 0;
     return SQLITE_OK;
   }
@@ -2669,8 +2901,8 @@ int sqlite3WalFindFrame(
   {
     u32 iRead2 = 0;
     u32 iTest;
-    assert( pWal->minFrame>0 );
+    assert( pWal->bShmUnreliable || pWal->minFrame>0 );
-    for(iTest=iLast; iTest>=pWal->minFrame; iTest--){
+    for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){
       if( walFramePgno(pWal, iTest)==pgno ){
         iRead2 = iTest;
         break;
@@ -3446,24 +3678,24 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){
   assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) );
 
   if( op==0 ){
-    if( pWal->exclusiveMode ){
+    if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){
-      pWal->exclusiveMode = 0;
+      pWal->exclusiveMode = WAL_NORMAL_MODE;
       if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){
-        pWal->exclusiveMode = 1;
+        pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
-      rc = pWal->exclusiveMode==0;
+      rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
     }else{
       /* Already in locking_mode=NORMAL */
       rc = 0;
     }
   }else if( op>0 ){
-    assert( pWal->exclusiveMode==0 );
+    assert( pWal->exclusiveMode==WAL_NORMAL_MODE );
     assert( pWal->readLock>=0 );
     walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
-    pWal->exclusiveMode = 1;
+    pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
     rc = 1;
   }else{
-    rc = pWal->exclusiveMode==0;
+    rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
   }
   return rc;
 }

test/wal2.test

@@ -122,8 +122,8 @@ do_test wal2-1.1 {
 } {4 10}
 
 set RECOVER [list                                      \
-  {0 1 lock exclusive}   {1 7 lock exclusive}          \
+  {0 1 lock exclusive}   {1 2 lock exclusive} {4 4 lock exclusive} \
-  {1 7 unlock exclusive} {0 1 unlock exclusive}        \
+  {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}  \
 ]
 set READ [list                                         \
   {4 1 lock shared}    {4 1 unlock shared}             \
@@ -393,8 +393,10 @@ tvfs delete
 set expected_locks [list]
 lappend expected_locks {1 1 lock exclusive}   ;# Lock checkpoint
 lappend expected_locks {0 1 lock exclusive}   ;# Lock writer
-lappend expected_locks {2 6 lock exclusive}   ;# Lock recovery & all aReadMark[]
+lappend expected_locks {2 1 lock exclusive}   ;# Lock recovery
-lappend expected_locks {2 6 unlock exclusive} ;# Unlock recovery & aReadMark[]
+lappend expected_locks {4 4 lock exclusive}   ;# Lock all aReadMark[]
+lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery 
+lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] 
 lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer
 lappend expected_locks {3 1 lock exclusive}   ;# Lock aReadMark[0]
 lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0]
@@ -615,8 +617,8 @@ do_test wal2-6.4.1 {
 } {}
 
 set RECOVERY {
-  {0 1 lock exclusive} {1 7 lock exclusive} 
+  {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive}
-  {1 7 unlock exclusive} {0 1 unlock exclusive}
+  {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}
 }
 set READMARK0_READ {
   {3 1 lock shared} {3 1 unlock shared}
@@ -1128,7 +1130,7 @@ if {$::tcl_platform(platform) == "unix"} {
   foreach {tn db_perm wal_perm shm_perm can_open can_read can_write} {
     2   00644   00644   00644   1   1   1
     3   00644   00400   00644   1   1   0
-    4   00644   00644   00400   1   0   0
+    4   00644   00644   00400   1   1   0
     5   00400   00644   00644   1   1   0
 
     7   00644   00000   00644   1   0   0

test/walro.test

@@ -101,10 +101,11 @@ do_multiclient_test tn {
     code1 { db close }
     list [file exists test.db-wal] [file exists test.db-shm]
   } {1 1}
+
   do_test 1.2.2 {
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
-    sql1 { SELECT * FROM t1 }
+    list [catch { sql1 { SELECT * FROM t1 } } msg] $msg
-  } {a b c d e f g h i j}
+  } {0 {a b c d e f g h i j}}
 
   do_test 1.2.3 {
     code1 { db close }
@@ -113,10 +114,10 @@ do_multiclient_test tn {
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {attempt to write a readonly database}}
+  } {0 {a b c d e f g h i j}}
   do_test 1.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_READONLY_RECOVERY}
+  } {SQLITE_OK}
 
   do_test 1.2.5 {
     file attributes test.db-shm -permissions rw-r--r--
@@ -138,11 +139,15 @@ do_multiclient_test tn {
   # Now check that if the readonly_shm option is not supplied, or if it
   # is set to zero, it is not possible to connect to the database without
   # read-write access to the shm.
+  # 
+  # UPDATE: os_unix.c now opens the *-shm file in readonly mode 
+  # automatically.
+  #
   do_test 1.3.1 {
     code1 { db close }
     code1 { sqlite3 db test.db }
     csql1 { SELECT * FROM t1 }
-  } {1 {unable to open database file}}
+  } {0 {a b c d e f g h i j k l}}
 
   # Also test that if the -shm file can be opened for read/write access,
   # it is not if readonly_shm=1 is present in the URI.
@@ -161,10 +166,10 @@ do_multiclient_test tn {
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {attempt to write a readonly database}}
+  } {0 {a b c d e f g h i j k l}}
   do_test 1.3.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_READONLY_RECOVERY}
+  } {SQLITE_OK}
 
   #-----------------------------------------------------------------------
   # Test cases 1.4.* check that checkpoints and log wraps don't prevent

test/walro2.test

@@ -0,0 +1,393 @@
+# 2011 May 09
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for using WAL databases in read-only mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix walro2
+
+# And only if the build is WAL-capable.
+#
+ifcapable !wal {
+  finish_test
+  return
+}
+
+proc copy_to_test2 {bZeroShm} {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  if {$bZeroShm} {
+    forcedelete test.db2-shm
+    set fd [open test.db2-shm w]
+    seek $fd [expr [file size test.db-shm]-1]
+    puts -nonewline $fd "\0"
+    close $fd
+  } else {
+    forcecopy test.db-shm test.db2-shm
+  }
+}
+
+foreach bZeroShm {0 1} {
+set TN [expr $bZeroShm+1]
+do_multiclient_test tn {
+  
+  # Close all connections and delete the database.
+  #
+  code1 { db close  }
+  code2 { db2 close }
+  code3 { db3 close }
+  forcedelete test.db
+  
+  # Do not run tests with the connections in the same process.
+  #
+  if {$tn==2} continue
+
+  foreach c {code1 code2 code3} {
+    $c {
+      sqlite3_shutdown
+      sqlite3_config_uri 1
+    }
+  }
+
+  do_test $TN.1.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      CREATE TABLE t1(x, y);
+      PRAGMA journal_mode = WAL;
+      INSERT INTO t1 VALUES('a', 'b');
+      INSERT INTO t1 VALUES('c', 'd');
+    }
+    file exists test.db-shm
+  } {1}
+
+  do_test $TN.1.2.1 {
+    copy_to_test2 $bZeroShm
+    code1 {
+      sqlite3 db file:test.db2?readonly_shm=1
+    }
+
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
+  do_test $TN.1.2.2 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
+
+  do_test $TN.1.3.1 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 { SELECT * FROM t1 }
+  } {a b c d}
+
+  do_test $TN.1.3.2 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
+
+  code1 { db close  }
+  code2 { db2 close }
+  code3 { db3 close }
+
+  do_test $TN.2.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      INSERT INTO t1 VALUES('e', 'f');
+      INSERT INTO t1 VALUES('g', 'h');
+    }
+    file exists test.db-shm
+  } {1}
+
+  do_test $TN.2.2 {
+    copy_to_test2 $bZeroShm
+    code1 {
+      sqlite3 db file:test.db2?readonly_shm=1
+    }
+    sql1 { 
+      BEGIN;
+      SELECT * FROM t1;
+    }
+  } {a b c d e f g h}
+
+  do_test $TN.2.3.1 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 { SELECT * FROM t1 }
+  } {a b c d e f g h}
+  do_test $TN.2.3.2 {
+    sql3 { INSERT INTO t1 VALUES('i', 'j') }
+    code3 { db3 close }
+    sql1 { COMMIT } 
+  } {}
+  do_test $TN.2.3.3 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h i j}
+
+
+  #-----------------------------------------------------------------------
+  # 3.1.*: That a readonly_shm connection can read a database file if both
+  #        the *-wal and *-shm files are zero bytes in size.
+  #
+  # 3.2.*: That it flushes the cache if, between transactions on a db with a
+  #        zero byte *-wal file, some other connection modifies the db, then
+  #        does "PRAGMA wal_checkpoint=truncate" to truncate the wal file
+  #        back to zero bytes in size.
+  #
+  # 3.3.*: That, if between transactions some other process wraps the wal
+  #        file, the readonly_shm client reruns recovery.
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+  do_test $TN.3.1.0 {
+    list [file exists test.db-wal] [file exists test.db-shm]
+  } {0 0}
+  do_test $TN.3.1.1 {
+    close [open test.db-wal w]
+    close [open test.db-shm w]
+    code1 {
+      sqlite3 db file:test.db?readonly_shm=1
+    }
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h}
+
+  do_test $TN.3.2.0 {
+    list [file size test.db-wal] [file size test.db-shm]
+  } {0 0}
+  do_test $TN.3.2.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate }
+    code2 { db2 close }
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h 1 2}
+  do_test $TN.3.2.2 {
+    list [file size test.db-wal] [file size test.db-shm]
+  } {0 32768}
+
+  do_test $TN.3.3.0 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      INSERT INTO t1 VALUES(3, 4);
+      INSERT INTO t1 VALUES(5, 6);
+      INSERT INTO t1 VALUES(7, 8);
+      INSERT INTO t1 VALUES(9, 10);
+    }
+    code2 { db2 close }
+    code1 { db close }
+    list [file size test.db-wal] [file size test.db-shm]
+  } [list [wal_file_size 4 1024] 32768]
+  do_test $TN.3.3.1 {
+    code1 { sqlite3 db file:test.db?readonly_shm=1 }
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10}
+  do_test $TN.3.3.2 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      PRAGMA wal_checkpoint; 
+      DELETE FROM t1;
+      INSERT INTO t1 VALUES('i', 'ii');
+    }
+    code2 { db2 close }
+    list [file size test.db-wal] [file size test.db-shm]
+  } [list [wal_file_size 4 1024] 32768]
+  do_test $TN.3.3.3 {
+    sql1 { SELECT * FROM t1 }
+  } {i ii}
+
+  #-----------------------------------------------------------------------
+  #
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test $TN.4.0 {
+    code1 { forcedelete test.db }
+    code1 { sqlite3 db test.db }
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('hello');
+      INSERT INTO t1 VALUES('world');
+    }
+
+    copy_to_test2 $bZeroShm
+
+    code1 { db close }
+  } {}
+
+  do_test $TN.4.1.1 {
+    code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
+    sql2 { SELECT * FROM t1 }
+  } {hello world}
+
+  do_test $TN.4.1.2 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 {
+      INSERT INTO t1 VALUES('!');
+      PRAGMA wal_checkpoint = truncate;
+    }
+    code3 { db3 close }
+  } {}
+  do_test $TN.4.1.3 {
+    sql2 { SELECT * FROM t1 }
+  } {hello world !}
+
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test $TN.4.2.1 {
+    code1 { sqlite3 db test.db }
+    sql1 {
+      INSERT INTO t1 VALUES('!');
+      INSERT INTO t1 VALUES('!');
+
+      PRAGMA cache_size = 10;
+      CREATE TABLE t2(x);
+
+      BEGIN;
+        WITH s(i) AS (
+          SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
+          )
+        INSERT INTO t2 SELECT randomblob(500) FROM s;
+        SELECT count(*) FROM t2;
+    } 
+  } {500}
+  do_test $TN.4.2.2 {
+    file size test.db-wal
+  } {461152}
+  do_test $TN.4.2.4 {
+    file_control_persist_wal db 1; db close
+
+    copy_to_test2 $bZeroShm
+    code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
+    sql2 {
+      SELECT * FROM t1;
+      SELECT count(*) FROM t2;
+    }
+  } {hello world ! ! 0}
+
+  #-----------------------------------------------------------------------
+  #
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test $TN.5.0 {
+    code1 { forcedelete test.db }
+    code1 { sqlite3 db test.db }
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('hello');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('!');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('hello');
+    }
+
+    copy_to_test2 $bZeroShm
+    
+    code1 { db close }
+  } {}
+
+  do_test $TN.5.1 {
+    code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
+    sql2 {
+      SELECT * FROM t1;
+    }
+  } {hello world ! world hello}
+
+  do_test $TN.5.2 {
+    code1 {
+      proc handle_read {op args} {
+        if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} {
+          set ::res2 [sql2 { SELECT * FROM t1 }]
+        }
+        puts "$msg xRead $args"
+        return "SQLITE_OK"
+      }
+      testvfs tvfs -fullshm 1
+
+      sqlite3 db file:test.db2?vfs=tvfs
+      db eval { SELECT * FROM sqlite_master }
+
+      tvfs filter xRead
+      tvfs script handle_read
+    }
+    sql1 {
+      PRAGMA wal_checkpoint = truncate;
+    }
+    code1 { set ::res2 }
+  } {hello world ! world hello}
+
+  do_test $TN.5.3 {
+    code1 { db close }
+    code1 { tvfs delete }
+  } {}
+
+  #-----------------------------------------------------------------------
+  #
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test $TN.6.1 {
+    code1 { forcedelete test.db }
+    code1 { sqlite3 db test.db }
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('hello');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('!');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('hello');
+    }
+
+    copy_to_test2 $bZeroShm
+    
+    code1 { db close }
+  } {}
+
+  do_test $TN.6.2 {
+    code1 {
+      set ::nRem 5
+      proc handle_read {op args} {
+        if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} {
+          incr ::nRem -1
+          if {$::nRem==0} {
+            code2 { sqlite3 db2 test.db2 }
+            sql2  { PRAGMA wal_checkpoint = truncate }
+          }
+        }
+        return "SQLITE_OK"
+      }
+      testvfs tvfs -fullshm 1
+
+      tvfs filter xRead
+      tvfs script handle_read
+
+      sqlite3 db file:test.db2?readonly_shm=1&vfs=tvfs
+      db eval { SELECT * FROM t1 }
+    }
+  } {hello world ! world hello}
+
+  do_test $TN.6.3 {
+    code1 { db close }
+    code1 { tvfs delete }
+  } {}
+}
+} ;# foreach bZeroShm
+
+finish_test

test/walrofault.test

@@ -0,0 +1,60 @@
+# 2011 May 09
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for using WAL databases in read-only mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix walro2
+
+# And only if the build is WAL-capable.
+#
+ifcapable !wal {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+sqlite3 db test.db
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(b);
+  PRAGMA journal_mode = wal;
+  INSERT INTO t1 VALUES('hello');
+  INSERT INTO t1 VALUES('world');
+  INSERT INTO t1 VALUES('!');
+  INSERT INTO t1 VALUES('world');
+  INSERT INTO t1 VALUES('hello');
+  PRAGMA cache_size = 10;
+  BEGIN;
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) 
+    INSERT INTO t1(b) SELECT randomblob(800) FROM s;
+} {wal}
+file_control_persist_wal db 1; db close
+faultsim_save_and_close
+
+do_faultsim_test 1 -faults oom* -prep {
+  catch { db close }
+  faultsim_restore
+  sqlite3 db file:test.db?readonly_shm=1
+} -body {
+  execsql { SELECT * FROM t1 }
+} -test {
+  faultsim_test_result {0 {hello world ! world hello}}
+}
+
+
+
+finish_test