From b5039fb7d3944049da6c48e83a6d7685b5cd6da2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 25 Oct 2017 23:28:13 +0000 Subject: [PATCH 01/33] Use extra locks to prevent a readonly_shm=1 process from connecting to a WAL-mode database if there are no writers. FossilOrigin-Name: 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 --- manifest | 17 +++++---- manifest.uuid | 2 +- src/os_unix.c | 96 ++++++++++++++++++++++++++++++------------------- src/sqlite.h.in | 1 + 4 files changed, 72 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index a593818258..bf7be72e5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase. -D 2017-10-25T19:18:33.009 +C Use\sextra\slocks\sto\sprevent\sa\sreadonly_shm=1\sprocess\sfrom\sconnecting\sto\sa\nWAL-mode\sdatabase\sif\sthere\sare\sno\swriters. +D 2017-10-25T23:28:13.771 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df +F src/os_unix.c bec568f6f89e57b987f5ca38ae90087687d8e18669cd37eceb48ca6b56a8fbfb F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f -F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7 +F src/sqlite.h.in ba9029e71a605bc5f236cd693abeb7920285785f2e1a92313ecf8fb1c9f52a86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9 @@ -1666,7 +1666,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1 -R de06ad2d264283eba96aba95f64ee6f5 +P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 +R 7c17d54c5cff309a6f13f49dd620b485 +T *branch * readonly-wal-recovery +T *sym-readonly-wal-recovery * +T -sym-trunk * U drh -Z 3b4d77a61659a96790923a30bda00b59 +Z 26f9af4fe6668c97b92039b43092c035 diff --git a/manifest.uuid b/manifest.uuid index ce16a7caf0..44570bf71c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 \ No newline at end of file +35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c33d21fd9b..dadfaf9908 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4145,6 +4145,7 @@ struct unixShm { */ #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ +#define UNIX_SHM_N_DMS 1000000000 /* Size of the DMS */ /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -4166,12 +4167,6 @@ static int unixShmSystemLock( pShmNode = pFile->pInode->pShmNode; assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); - /* Shared locks never span more than one byte */ - assert( n==1 || lockType!=F_RDLCK ); - - /* Locks are within range */ - assert( n>=1 && n<=SQLITE_SHM_NLOCK ); - if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); @@ -4186,36 +4181,44 @@ static int unixShmSystemLock( /* Update the global lock state and do debug tracing */ #ifdef SQLITE_DEBUG - { u16 mask; - OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; + if( ofst=1 && n<=SQLITE_SHM_NLOCK ); + + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; + } }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } } - }else{ - if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); - } - } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); } #endif @@ -4389,17 +4392,38 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. + /* Do not allow a read-only process to connect if there are no + ** writers, because a read-only process is unable to recover the + ** shm file following a system crash. */ rc = SQLITE_OK; + if( pShmNode->isReadonly ){ + if( !unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, UNIX_SHM_N_DMS) ){ + rc = SQLITE_CANTOPEN_DIRTYWAL; + } + } + + /* If we are able to grab the dead-man switch, that means this is the + ** first (write-enable) process to connect to the database. In that + ** case, truncate the shm file because the contents found on disk might + ** be invalid leftovers from a system crash. The shm will be rebuilt + */ if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } } + + /* Acquires locks to tell other processes that a this process is + ** running and therefore the shm is valid they do not need to run + ** recovery. + */ if( rc==SQLITE_OK ){ + unsigned r; rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + sqlite3_randomness(sizeof(r), &r); + r = 1 + (r%(UNIX_SHM_N_DMS-1)); + (void)unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS+r, 1); } if( rc ) goto shm_open_err; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9ff366304f..0c0ed25b30 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -508,6 +508,7 @@ int sqlite3_exec( #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) From 9181ae990a3b8623d27813209e6649bf76565545 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 26 Oct 2017 17:05:22 +0000 Subject: [PATCH 02/33] Instead of extra locks, use F_GETLK to ensure that readonly_shm clients cannot connect to a wal-mode database if there are no writers. FossilOrigin-Name: 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 --- manifest | 17 ++++---- manifest.uuid | 2 +- src/os_unix.c | 107 ++++++++++++++++++++++---------------------------- 3 files changed, 56 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index bf7be72e5e..c648e5a856 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sextra\slocks\sto\sprevent\sa\sreadonly_shm=1\sprocess\sfrom\sconnecting\sto\sa\nWAL-mode\sdatabase\sif\sthere\sare\sno\swriters. -D 2017-10-25T23:28:13.771 +C Instead\sof\sextra\slocks,\suse\sF_GETLK\sto\sensure\sthat\sreadonly_shm\sclients\scannot\nconnect\sto\sa\swal-mode\sdatabase\sif\sthere\sare\sno\swriters. +D 2017-10-26T17:05:22.656 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c bec568f6f89e57b987f5ca38ae90087687d8e18669cd37eceb48ca6b56a8fbfb +F src/os_unix.c 8103f60342c65d501b4e58b381796648d6584b4814ffee79cd3a6e0c12fb6545 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1666,10 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18 -R 7c17d54c5cff309a6f13f49dd620b485 -T *branch * readonly-wal-recovery -T *sym-readonly-wal-recovery * -T -sym-trunk * -U drh -Z 26f9af4fe6668c97b92039b43092c035 +P 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 +R 87551e9df4f45e3704d812374bd0d192 +U dan +Z a113fb74c5589c985f30184211e03e42 diff --git a/manifest.uuid b/manifest.uuid index 44570bf71c..1815e9a333 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 \ No newline at end of file +5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index dadfaf9908..f689ff75fa 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4145,7 +4145,6 @@ struct unixShm { */ #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ -#define UNIX_SHM_N_DMS 1000000000 /* Size of the DMS */ /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -4167,6 +4166,12 @@ static int unixShmSystemLock( pShmNode = pFile->pInode->pShmNode; assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); + /* Shared locks never span more than one byte */ + assert( n==1 || lockType!=F_RDLCK ); + + /* Locks are within range */ + assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); @@ -4181,44 +4186,36 @@ static int unixShmSystemLock( /* Update the global lock state and do debug tracing */ #ifdef SQLITE_DEBUG - if( ofst=1 && n<=SQLITE_SHM_NLOCK ); - - OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; - } + { u16 mask; + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; }else{ - if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); - } + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); + }else{ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } + } + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); } #endif @@ -4392,38 +4389,30 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - /* Do not allow a read-only process to connect if there are no - ** writers, because a read-only process is unable to recover the - ** shm file following a system crash. - */ + /* Check to see if another process is holding the dead-man switch. + ** For a readonly_shm client, if no other process holds the DMS lock, + ** the file cannot be opened and SQLITE_CANTOPEN_DIRTYWAL is returned. + ** Or, for a read-write connection, if no other process holds a + ** DMS lock the file is truncated to zero bytes in size. */ rc = SQLITE_OK; if( pShmNode->isReadonly ){ - if( !unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, UNIX_SHM_N_DMS) ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lockInfo)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ rc = SQLITE_CANTOPEN_DIRTYWAL; } - } - - /* If we are able to grab the dead-man switch, that means this is the - ** first (write-enable) process to connect to the database. In that - ** case, truncate the shm file because the contents found on disk might - ** be invalid leftovers from a system crash. The shm will be rebuilt - */ - if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + }else if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } } - - /* Acquires locks to tell other processes that a this process is - ** running and therefore the shm is valid they do not need to run - ** recovery. - */ if( rc==SQLITE_OK ){ - unsigned r; rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); - sqlite3_randomness(sizeof(r), &r); - r = 1 + (r%(UNIX_SHM_N_DMS-1)); - (void)unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS+r, 1); } if( rc ) goto shm_open_err; } From ab04eff809c0ee16a913e469ce51acdfcaf20c5d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 26 Oct 2017 17:34:50 +0000 Subject: [PATCH 03/33] Fix an error in the previous commit on this branch. FossilOrigin-Name: f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- test/walro.test | 13 +++++++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c648e5a856..ebf38789b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Instead\sof\sextra\slocks,\suse\sF_GETLK\sto\sensure\sthat\sreadonly_shm\sclients\scannot\nconnect\sto\sa\swal-mode\sdatabase\sif\sthere\sare\sno\swriters. -D 2017-10-26T17:05:22.656 +C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch. +D 2017-10-26T17:34:50.823 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 8103f60342c65d501b4e58b381796648d6584b4814ffee79cd3a6e0c12fb6545 +F src/os_unix.c 9bf0c1b7156cbcd2ec32e557cffa319251e1ffb1515d923a2dd2d8eee69b4ee4 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1526,7 +1526,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 -F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160 +F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8 -R 87551e9df4f45e3704d812374bd0d192 +P 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 +R 49c1c99b22f8e84ff20a7afe5d1d547b U dan -Z a113fb74c5589c985f30184211e03e42 +Z 07348874af35473af5ead94c9a86c9df diff --git a/manifest.uuid b/manifest.uuid index 1815e9a333..3da837219d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 \ No newline at end of file +f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f689ff75fa..7720a255ac 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4401,7 +4401,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ lock.l_start = UNIX_SHM_DMS; lock.l_len = 1; lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lockInfo)!=0 ) { + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { rc = SQLITE_IOERR_LOCK; }else if( lock.l_type==F_UNLCK ){ rc = SQLITE_CANTOPEN_DIRTYWAL; diff --git a/test/walro.test b/test/walro.test index f46e44d4cb..09a33bbd3e 100644 --- a/test/walro.test +++ b/test/walro.test @@ -101,10 +101,11 @@ do_multiclient_test tn { code1 { db close } list [file exists test.db-wal] [file exists test.db-shm] } {1 1} + do_test 1.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } - sql1 { SELECT * FROM t1 } - } {a b c d e f g h i j} + list [catch { sql1 { SELECT * FROM t1 } } msg] $msg + } {1 {unable to open database file}} do_test 1.2.3 { code1 { db close } @@ -113,10 +114,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {attempt to write a readonly database}} + } {1 {unable to open database file}} do_test 1.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_READONLY_RECOVERY} + } {SQLITE_CANTOPEN} do_test 1.2.5 { file attributes test.db-shm -permissions rw-r--r-- @@ -161,10 +162,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {attempt to write a readonly database}} + } {1 {unable to open database file}} do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_READONLY_RECOVERY} + } {SQLITE_CANTOPEN} #----------------------------------------------------------------------- # Test cases 1.4.* check that checkpoints and log wraps don't prevent From 176b2a916bfa2d6577638713d4e3e3264132f122 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Nov 2017 06:59:19 +0000 Subject: [PATCH 04/33] Fix a race condition in os_unix.c that might allow a client to use a *-shm file corrupted by a power failure if another client fails between locking the *-shm file and truncating it to zero bytes. FossilOrigin-Name: d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 54 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ebf38789b5..91f071548a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch. -D 2017-10-26T17:34:50.823 +C Fix\sa\srace\scondition\sin\sos_unix.c\sthat\smight\sallow\sa\sclient\sto\suse\sa\s*-shm\nfile\scorrupted\sby\sa\spower\sfailure\sif\sanother\sclient\sfails\sbetween\slocking\sthe\n*-shm\sfile\sand\struncating\sit\sto\szero\sbytes. +D 2017-11-01T06:59:19.745 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 9bf0c1b7156cbcd2ec32e557cffa319251e1ffb1515d923a2dd2d8eee69b4ee4 +F src/os_unix.c 9137cfdb42e83f4fb599aa2b1c6996f368aacbb410bde53b534e766a61ba65ca F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923 -R 49c1c99b22f8e84ff20a7afe5d1d547b +P f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 +R 37c523032e6b14675e1621a01cbce4c1 U dan -Z 07348874af35473af5ead94c9a86c9df +Z e56573f2b7ef574fffc6cf13c6dfb22b diff --git a/manifest.uuid b/manifest.uuid index 3da837219d..46c47f2d96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0 \ No newline at end of file +d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7720a255ac..b94d417f49 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4372,6 +4372,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ + struct flock lock; int openFlags = O_RDWR | O_CREAT; if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ openFlags = O_RDONLY; @@ -4389,29 +4390,46 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - /* Check to see if another process is holding the dead-man switch. - ** For a readonly_shm client, if no other process holds the DMS lock, - ** the file cannot be opened and SQLITE_CANTOPEN_DIRTYWAL is returned. - ** Or, for a read-write connection, if no other process holds a - ** DMS lock the file is truncated to zero bytes in size. */ + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ rc = SQLITE_OK; - if( pShmNode->isReadonly ){ - struct flock lock; - lock.l_whence = SEEK_SET; - lock.l_start = UNIX_SHM_DMS; - lock.l_len = 1; - lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { - rc = SQLITE_IOERR_LOCK; - }else if( lock.l_type==F_UNLCK ){ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ rc = SQLITE_CANTOPEN_DIRTYWAL; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); + } } - }else if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; } + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); } if( rc ) goto shm_open_err; From 92c02da33ec2717828eba13400355d43a7a6395f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Nov 2017 20:59:28 +0000 Subject: [PATCH 05/33] If a readonly_shm connection cannot map the *-shm file because no other process is holding the DMS lock, have it read from the database file only, ignoring any content in the wal file. FossilOrigin-Name: ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 --- manifest | 15 ++++--- manifest.uuid | 2 +- src/os_unix.c | 114 ++++++++++++++++++++++++++++------------------- src/wal.c | 24 +++++++--- test/walro2.test | 87 ++++++++++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 59 deletions(-) create mode 100644 test/walro2.test diff --git a/manifest b/manifest index a6e4cc1fc1..45c7b16d67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2017-11-01T07:06:41.244 +C If\sa\sreadonly_shm\sconnection\scannot\smap\sthe\s*-shm\sfile\sbecause\sno\sother\nprocess\sis\sholding\sthe\sDMS\slock,\shave\sit\sread\sfrom\sthe\sdatabase\sfile\sonly,\nignoring\sany\scontent\sin\sthe\swal\sfile. +D 2017-11-01T20:59:28.295 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 9137cfdb42e83f4fb599aa2b1c6996f368aacbb410bde53b534e766a61ba65ca +F src/os_unix.c e376adf6014df7d1a73faaaa6c87e6eb9b299b157a58cccff02fad8abc943fe7 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579 +F src/wal.c 1521bdcfe9a536752a339c91b63ca0d226d8d266636391aac93537fdea8657fc F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,6 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a +F test/walro2.test e2cd102cafceafaf19c56d55e55222fdd26d7621d7ef42b0eaf35c06c5bb3d19 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1667,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de -R 46264ab14606a934f99609a277e3ff11 +P 985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9 +R fcc128b567671c56015745eca2c28340 U dan -Z 7bfe9089058e9a41a8fafe2fd8ba1cb5 +Z e9ea9ea441f189aa52edada39fcf2f83 diff --git a/manifest.uuid b/manifest.uuid index 48096a84ae..ebf42c65a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9 \ No newline at end of file +ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b94d417f49..e90e335cb7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4108,6 +4108,7 @@ struct unixShmNode { int szRegion; /* Size of shared-memory regions */ u16 nRegion; /* Size of array apRegion */ u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ @@ -4270,6 +4271,64 @@ static void unixShmPurge(unixFile *pFd){ } } +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1. +*/ +static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + struct flock lock; + int rc = SQLITE_OK; + + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTLOCK; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + } + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + } + return rc; +} + /* ** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. @@ -4308,7 +4367,7 @@ static void unixShmPurge(unixFile *pFd){ static int unixOpenSharedMemory(unixFile *pDbFd){ struct unixShm *p = 0; /* The connection to be opened */ struct unixShmNode *pShmNode; /* The underlying mmapped file */ - int rc; /* Result code */ + int rc = SQLITE_OK; /* Result code */ unixInodeInfo *pInode; /* The inode of fd */ char *zShmFilename; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ @@ -4372,7 +4431,6 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - struct flock lock; int openFlags = O_RDWR | O_CREAT; if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ openFlags = O_RDONLY; @@ -4389,50 +4447,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** the original owner will not be able to connect. */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - - /* Use F_GETLK to determine the locks other processes are holding - ** on the DMS byte. If it indicates that another process is holding - ** a SHARED lock, then this process may also take a SHARED lock - ** and proceed with opening the *-shm file. - ** - ** Or, if no other process is holding any lock, then this process - ** is the first to open it. In this case take an EXCLUSIVE lock on the - ** DMS byte and truncate the *-shm file to zero bytes in size. Then - ** downgrade to a SHARED lock on the DMS byte. - ** - ** If another process is holding an EXCLUSIVE lock on the DMS byte, - ** return SQLITE_BUSY to the caller (it will try again). An earlier - ** version of this code attempted the SHARED lock at this point. But - ** this introduced a subtle race condition: if the process holding - ** EXCLUSIVE failed just before truncating the *-shm file, then this - ** process might open and use the *-shm file without truncating it. - ** And if the *-shm file has been corrupted by a power failure or - ** system crash, the database itself may also become corrupt. */ - rc = SQLITE_OK; - lock.l_whence = SEEK_SET; - lock.l_start = UNIX_SHM_DMS; - lock.l_len = 1; - lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { - rc = SQLITE_IOERR_LOCK; - }else if( lock.l_type==F_UNLCK ){ - if( pShmNode->isReadonly ){ - rc = SQLITE_CANTOPEN_DIRTYWAL; - }else{ - rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); - if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } - } - }else if( lock.l_type==F_WRLCK ){ - rc = SQLITE_BUSY; - } - if( rc==SQLITE_OK ){ - assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); - rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err; } } @@ -4456,7 +4473,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: @@ -4508,6 +4525,11 @@ static int unixShmMap( p = pDbFd->pShm; pShmNode = p->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); diff --git a/src/wal.c b/src/wal.c index 19c9ea0a08..1a11eb31e0 100644 --- a/src/wal.c +++ b/src/wal.c @@ -575,9 +575,11 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); - if( rc==SQLITE_READONLY ){ + if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; - rc = SQLITE_OK; + if( rc==SQLITE_READONLY ){ + rc = SQLITE_OK; + } } } } @@ -2084,6 +2086,14 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); if( rc!=SQLITE_OK ){ + if( rc==SQLITE_READONLY_CANTLOCK +#ifdef SQLITE_ENABLE_SNAPSHOT + && pWal->pSnapshot==0 +#endif + ){ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + rc = SQLITE_OK; + } return rc; }; assert( page0 || pWal->writeLock==0 ); @@ -2259,8 +2269,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } } - pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame + assert( pWal->nWiData>0 ); + assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) ); + pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0; + if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame) #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) @@ -2272,7 +2284,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(0)); walShmBarrier(pWal); if( rc==SQLITE_OK ){ - if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + if( pInfo + && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ /* It is not safe to allow the reader to continue here if frames ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, diff --git a/test/walro2.test b/test/walro2.test new file mode 100644 index 0000000000..2337776f27 --- /dev/null +++ b/test/walro2.test @@ -0,0 +1,87 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walro + +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} + +# And only if the build is WAL-capable. +# +ifcapable !wal { + finish_test + return +} + +do_multiclient_test tn { + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + + do_test 1.1 { + code2 { sqlite3 db2 test.db } + sql2 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); + } + file exists test.db-shm + } {1} + + do_test 1.2 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + code1 { + sqlite3 db file:test.db2?readonly_shm=1 + } + + sql1 { SELECT * FROM t1 } + } {} + + do_test 1.3.1 { + code3 { sqlite3 db3 test.db2 } + sql3 { SELECT * FROM t1 } + } {a b c d} + + do_test 1.3.2 { + sql1 { SELECT * FROM t1 } + } {a b c d} + +} + +finish_test From dea5ce36f57f83b4c6c78154c088d67ec8c5c594 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Nov 2017 11:12:03 +0000 Subject: [PATCH 06/33] Avoid locking shm-lock WAL_READ_LOCK(0) during recovery. Doing this allows recovery to proceed while a readonly_shm connection in unlocked mode has an ongoing read transaction. FossilOrigin-Name: 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 14 ++++++++++---- test/walro2.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 45c7b16d67..f3b7f8abe0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sreadonly_shm\sconnection\scannot\smap\sthe\s*-shm\sfile\sbecause\sno\sother\nprocess\sis\sholding\sthe\sDMS\slock,\shave\sit\sread\sfrom\sthe\sdatabase\sfile\sonly,\nignoring\sany\scontent\sin\sthe\swal\sfile. -D 2017-11-01T20:59:28.295 +C Avoid\slocking\sshm-lock\sWAL_READ_LOCK(0)\sduring\srecovery.\sDoing\sthis\sallows\nrecovery\sto\sproceed\swhile\sa\sreadonly_shm\sconnection\sin\sunlocked\smode\shas\san\nongoing\sread\stransaction. +D 2017-11-02T11:12:03.045 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 1521bdcfe9a536752a339c91b63ca0d226d8d266636391aac93537fdea8657fc +F src/wal.c 38480e7bdc697cf88a13a22ffe60f7bd761bc02b45f7a323f1bb9e61a136b3ae F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a -F test/walro2.test e2cd102cafceafaf19c56d55e55222fdd26d7621d7ef42b0eaf35c06c5bb3d19 +F test/walro2.test 23fea1e7abae13072b0640ef846d32080b2fc435658d4c4eb9db266b07b33776 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9 -R fcc128b567671c56015745eca2c28340 +P ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 +R 5519605a9af9008f6aa3aade00e8ea39 U dan -Z e9ea9ea441f189aa52edada39fcf2f83 +Z aa82fa216e4468716131c3fe0543b69b diff --git a/manifest.uuid b/manifest.uuid index ebf42c65a3..08fbfd4ed2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 \ No newline at end of file +5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 1a11eb31e0..28fade96b0 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1101,7 +1101,6 @@ static int walIndexRecover(Wal *pWal){ i64 nSize; /* Size of log file */ u32 aFrameCksum[2] = {0, 0}; int iLock; /* Lock offset to lock for checkpoint */ - int nLock; /* Number of locks to hold */ /* Obtain an exclusive lock on all byte in the locking range not already ** locked by the caller. The caller is guaranteed to have locked the @@ -1114,11 +1113,17 @@ static int walIndexRecover(Wal *pWal){ assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; - nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); + rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + if( rc==SQLITE_OK ){ + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc!=SQLITE_OK ){ + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + } + } if( rc ){ return rc; } + WALTRACE(("WAL%p: recovery begin...\n", pWal)); memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); @@ -1256,7 +1261,8 @@ finished: recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); - walUnlockExclusive(pWal, iLock, nLock); + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); return rc; } diff --git a/test/walro2.test b/test/walro2.test index 2337776f27..fb41d17f79 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -82,6 +82,37 @@ do_multiclient_test tn { sql1 { SELECT * FROM t1 } } {a b c d} + code1 { db close } + code2 { db2 close } + code3 { db3 close } + + do_test 2.1 { + code2 { sqlite3 db2 test.db } + sql2 { + INSERT INTO t1 VALUES('e', 'f'); + INSERT INTO t1 VALUES('g', 'h'); + } + file exists test.db-shm + } {1} + + do_test 2.2 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + code1 { + sqlite3 db file:test.db2?readonly_shm=1 + } + sql1 { + BEGIN; + SELECT * FROM t1; + } + } {a b c d} + + do_test 2.3.1 { + code3 { sqlite3 db3 test.db2 } + sql3 { SELECT * FROM t1 } + } {a b c d e f g h} + } finish_test From 44c8a97e019bfb791fd9b330212e2375d7516849 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Nov 2017 18:57:46 +0000 Subject: [PATCH 07/33] Fix test cases in wal2.test broken by the locking change in the previous commit. FossilOrigin-Name: f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal2.test | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f3b7f8abe0..5ad595579c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\slocking\sshm-lock\sWAL_READ_LOCK(0)\sduring\srecovery.\sDoing\sthis\sallows\nrecovery\sto\sproceed\swhile\sa\sreadonly_shm\sconnection\sin\sunlocked\smode\shas\san\nongoing\sread\stransaction. -D 2017-11-02T11:12:03.045 +C Fix\stest\scases\sin\swal2.test\sbroken\sby\sthe\slocking\schange\sin\sthe\sprevious\ncommit. +D 2017-11-02T18:57:46.036 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1501,7 +1501,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef +F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3 -R 5519605a9af9008f6aa3aade00e8ea39 +P 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d +R 30693e487c23aaa3c080ebb269e4e38f U dan -Z aa82fa216e4468716131c3fe0543b69b +Z a5242a9fca30f5b5641106b18cdb01ae diff --git a/manifest.uuid b/manifest.uuid index 08fbfd4ed2..a8bcfdb1fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d \ No newline at end of file +f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f \ No newline at end of file diff --git a/test/wal2.test b/test/wal2.test index bc170ad3ff..ddff71aed4 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -122,8 +122,8 @@ do_test wal2-1.1 { } {4 10} set RECOVER [list \ - {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ + {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ ] set READ [list \ {4 1 lock shared} {4 1 unlock shared} \ @@ -393,8 +393,10 @@ tvfs delete set expected_locks [list] lappend expected_locks {1 1 lock exclusive} ;# Lock checkpoint lappend expected_locks {0 1 lock exclusive} ;# Lock writer -lappend expected_locks {2 6 lock exclusive} ;# Lock recovery & all aReadMark[] -lappend expected_locks {2 6 unlock exclusive} ;# Unlock recovery & aReadMark[] +lappend expected_locks {2 1 lock exclusive} ;# Lock recovery +lappend expected_locks {4 4 lock exclusive} ;# Lock all aReadMark[] +lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery +lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer lappend expected_locks {3 1 lock exclusive} ;# Lock aReadMark[0] lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0] @@ -615,8 +617,8 @@ do_test wal2-6.4.1 { } {} set RECOVERY { - {0 1 lock exclusive} {1 7 lock exclusive} - {1 7 unlock exclusive} {0 1 unlock exclusive} + {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} } set READMARK0_READ { {3 1 lock shared} {3 1 unlock shared} From 11caf4f4b73736f6b209d591d652c7e0398aa024 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Nov 2017 18:10:03 +0000 Subject: [PATCH 08/33] In cases where a readonly_shm client cannot take the DMS lock on the *-shm file, have it parse the wal file and create a wal-index to access it in heap memory. FossilOrigin-Name: 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 --- manifest | 16 ++--- manifest.uuid | 2 +- src/wal.c | 176 +++++++++++++++++++++++++++++++++++++++++++---- test/walro.test | 10 +-- test/walro2.test | 19 +++-- 5 files changed, 192 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 5ad595579c..7e5b9e5a86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scases\sin\swal2.test\sbroken\sby\sthe\slocking\schange\sin\sthe\sprevious\ncommit. -D 2017-11-02T18:57:46.036 +C In\scases\swhere\sa\sreadonly_shm\sclient\scannot\stake\sthe\sDMS\slock\son\sthe\s*-shm\nfile,\shave\sit\sparse\sthe\swal\sfile\sand\screate\sa\swal-index\sto\saccess\sit\sin\sheap\nmemory. +D 2017-11-04T18:10:03.528 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 38480e7bdc697cf88a13a22ffe60f7bd761bc02b45f7a323f1bb9e61a136b3ae +F src/wal.c 2b287b5250e89d548c6bbd1d204d0db41046bb3984b9b4a79fc84e22359f1beb F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1526,8 +1526,8 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 -F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a -F test/walro2.test 23fea1e7abae13072b0640ef846d32080b2fc435658d4c4eb9db266b07b33776 +F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 +F test/walro2.test 611ceebd190edeca9bf39e5068cbc864f15294371b4acf9ee837db477840af54 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d -R 30693e487c23aaa3c080ebb269e4e38f +P f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f +R 10fc0a1645ce290a2a283e11360fe887 U dan -Z a5242a9fca30f5b5641106b18cdb01ae +Z 79ea7f3203670e23f752126d91be88fd diff --git a/manifest.uuid b/manifest.uuid index a8bcfdb1fe..e8b4f82532 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f \ No newline at end of file +18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 28fade96b0..17e253a8b6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -455,6 +455,7 @@ struct Wal { u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ + u8 bUnlocked; WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ @@ -1270,13 +1271,14 @@ recovery_error: ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bUnlocked ){ int i; for(i=0; inWiData; i++){ sqlite3_free((void *)pWal->apWiData[i]); pWal->apWiData[i] = 0; } - }else{ + } + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmUnmap(pWal->pDbFd, isDelete); } } @@ -2091,15 +2093,13 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); + if( rc==SQLITE_READONLY_CANTLOCK ){ + assert( page0==0 && pWal->writeLock==0 ); + pWal->bUnlocked = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else if( rc!=SQLITE_OK ){ - if( rc==SQLITE_READONLY_CANTLOCK -#ifdef SQLITE_ENABLE_SNAPSHOT - && pWal->pSnapshot==0 -#endif - ){ - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - rc = SQLITE_OK; - } return rc; }; assert( page0 || pWal->writeLock==0 ); @@ -2116,7 +2116,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( pWal->bUnlocked==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; @@ -2146,6 +2146,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; } + if( pWal->bUnlocked ){ + if( rc!=SQLITE_OK ){ + walIndexClose(pWal, 0); + } + pWal->exclusiveMode = WAL_NORMAL_MODE; + } return rc; } @@ -2156,6 +2162,144 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ #define WAL_RETRY (-1) +/* +** Open an "unlocked" transaction. An unlocked transaction is a read +** transaction used by a read-only client in cases where the *-shm +** file cannot be mapped and its contents cannot be trusted. It is +** assumed that the *-wal file has been read and that a wal-index +** constructed in heap memory is currently available in Wal.apWiData[]. +** +** If this function returns SQLITE_OK, then the read transaction has +** been successfully opened. In this case output variable (*pChanged) +** is set to true before returning if the caller should discard the +** contents of the page cache before proceeding. Or, if it returns +** WAL_RETRY, then the heap memory wal-index has been discarded and +** the caller should retry opening the read transaction from the +** beginning (including attempting to map the *-shm file). +** +** If an error occurs, an SQLite error code is returned. +*/ +static int walBeginUnlocked(Wal *pWal, int *pChanged){ + i64 szWal; /* Size of wal file on disk in bytes */ + i64 iOffset; /* Current offset when reading wal file */ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + volatile void *pDummy; /* Dummy argument for xShmMap */ + int rc; /* Return code */ + u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ + + assert( pWal->bUnlocked ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Take WAL_READ_LOCK(0). This has the effect of preventing any + ** live clients from running a checkpoint, but does not stop them + ** from running recovery. */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + if( rc!=SQLITE_OK ){ + return (rc==SQLITE_BUSY ? WAL_RETRY : rc); + } + pWal->readLock = 0; + + /* Try to map the *-shm file again. If it succeeds this time, then + ** a non-readonly_shm connection has already connected to the database. + ** In this case, start over with opening the transaction. + ** + ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint + ** from taking place. But it does not prevent the wal from being wrapped + ** if a checkpoint has already taken place. This means that if another + ** client is connected at this point, it may have already checkpointed + ** the entire wal. In that case it would not be safe to continue with + ** the unlocked transaction, as the other client may overwrite wal + ** frames that this client is still using. */ + rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + if( rc!=SQLITE_READONLY_CANTLOCK ){ + assert( rc!=SQLITE_OK ); + rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); + goto begin_unlocked_out; + } + + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); + if( rc!=SQLITE_OK || (szWalhdr.mxFrame==0) ){ + /* If the wal file is too small to contain a wal-header and the + ** wal-index header has mxFrame==0, then it must be safe to proceed + ** reading the database file only. However, the page cache cannot + ** be trusted, as a read/write connection may have connected, written + ** the db, run a checkpoint, truncated the wal file and disconnected + ** since this client's last read transaction. */ + *pChanged = 1; + goto begin_unlocked_out; + } + + /* Check the salt keys at the start of the wal file still match. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto begin_unlocked_out; + } + if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + rc = WAL_RETRY; + goto begin_unlocked_out; + } + + /* Allocate a buffer to read frames into */ + szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( aFrame==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto begin_unlocked_out; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; + aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; + for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); + iOffset+szFrame<=szWal; + iOffset+=szFrame + ){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_SHORT_READ ){ + /* If this branch is taken, some other client has truncated the + ** *-wal file since the call to sqlite3OsFileSize() above. This + ** indicates that a read-write client has connected to the system. + ** So retry opening this read transaction. */ + rc = WAL_RETRY; + } + break; + } + if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + rc = WAL_RETRY; + break; + } + } + pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; + pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; + + begin_unlocked_out: + sqlite3_free(aFrame); + if( rc!=SQLITE_OK ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void*)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + pWal->bUnlocked = 0; + sqlite3WalEndReadTransaction(pWal); + *pChanged = 1; + } + return rc; +} + /* ** Attempt to start a read transaction. This might fail due to a race or ** other transient condition. When that happens, it returns WAL_RETRY to @@ -2244,7 +2388,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } if( !useWal ){ - rc = walIndexReadHdr(pWal, pChanged); + assert( rc==SQLITE_OK ); + if( pWal->bUnlocked==0 ){ + rc = walIndexReadHdr(pWal, pChanged); + } if( rc==SQLITE_BUSY ){ /* If there is not a recovery running in another thread or process ** then convert BUSY errors to WAL_RETRY. If recovery is known to @@ -2273,6 +2420,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( rc!=SQLITE_OK ){ return rc; } + else if( pWal->bUnlocked ){ + return walBeginUnlocked(pWal, pChanged); + } } assert( pWal->nWiData>0 ); @@ -2626,7 +2776,7 @@ int sqlite3WalFindFrame( ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || pWal->readLock==0 ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bUnlocked==0) ){ *piRead = 0; return SQLITE_OK; } diff --git a/test/walro.test b/test/walro.test index 09a33bbd3e..150344e151 100644 --- a/test/walro.test +++ b/test/walro.test @@ -105,7 +105,7 @@ do_multiclient_test tn { do_test 1.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } list [catch { sql1 { SELECT * FROM t1 } } msg] $msg - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j}} do_test 1.2.3 { code1 { db close } @@ -114,10 +114,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j}} do_test 1.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_CANTOPEN} + } {SQLITE_OK} do_test 1.2.5 { file attributes test.db-shm -permissions rw-r--r-- @@ -162,10 +162,10 @@ do_multiclient_test tn { file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j k l}} do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_CANTOPEN} + } {SQLITE_OK} #----------------------------------------------------------------------- # Test cases 1.4.* check that checkpoints and log wraps don't prevent diff --git a/test/walro2.test b/test/walro2.test index fb41d17f79..ee4a341e24 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -62,7 +62,7 @@ do_multiclient_test tn { file exists test.db-shm } {1} - do_test 1.2 { + do_test 1.2.1 { forcecopy test.db test.db2 forcecopy test.db-wal test.db2-wal forcecopy test.db-shm test.db2-shm @@ -71,7 +71,10 @@ do_multiclient_test tn { } sql1 { SELECT * FROM t1 } - } {} + } {a b c d} + do_test 1.2.2 { + sql1 { SELECT * FROM t1 } + } {a b c d} do_test 1.3.1 { code3 { sqlite3 db3 test.db2 } @@ -106,13 +109,21 @@ do_multiclient_test tn { BEGIN; SELECT * FROM t1; } - } {a b c d} + } {a b c d e f g h} do_test 2.3.1 { code3 { sqlite3 db3 test.db2 } sql3 { SELECT * FROM t1 } } {a b c d e f g h} - + do_test 2.3.2 { + sql3 { INSERT INTO t1 VALUES('i', 'j') } + code3 { db3 close } + sql1 { COMMIT } + } {} + breakpoint + do_test 2.3.3 { + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j} } finish_test From cbd3321978bd66bb53f87abe4e25c1f57b087652 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Nov 2017 21:06:35 +0000 Subject: [PATCH 09/33] Add further tests for the code added on this branch. FossilOrigin-Name: a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f --- manifest | 14 ++++----- manifest.uuid | 2 +- src/wal.c | 8 ++++- test/walro2.test | 76 ++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 89 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7e5b9e5a86..84f5558cad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\scases\swhere\sa\sreadonly_shm\sclient\scannot\stake\sthe\sDMS\slock\son\sthe\s*-shm\nfile,\shave\sit\sparse\sthe\swal\sfile\sand\screate\sa\swal-index\sto\saccess\sit\sin\sheap\nmemory. -D 2017-11-04T18:10:03.528 +C Add\sfurther\stests\sfor\sthe\scode\sadded\son\sthis\sbranch. +D 2017-11-04T21:06:35.734 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 2b287b5250e89d548c6bbd1d204d0db41046bb3984b9b4a79fc84e22359f1beb +F src/wal.c 0b3c6b805fc1cf288a7c63b1ac0f78dcc0ad6a54b5b0d72fb0992b16360e7647 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 -F test/walro2.test 611ceebd190edeca9bf39e5068cbc864f15294371b4acf9ee837db477840af54 +F test/walro2.test 811ab176ab8571c59c2aac889fdacc7cff8d7a1ceb083796540c2886620a599f F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f -R 10fc0a1645ce290a2a283e11360fe887 +P 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 +R 5345a5078e9be9411c860b392d7c8c77 U dan -Z 79ea7f3203670e23f752126d91be88fd +Z 9992407e85299b3993fc8148a1281ba1 diff --git a/manifest.uuid b/manifest.uuid index e8b4f82532..ce4b2816dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 \ No newline at end of file +a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 17e253a8b6..d8612ec850 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2253,6 +2253,10 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ } aData = &aFrame[WAL_FRAME_HDRSIZE]; + /* Check to see if a complete transaction has been appended to the + ** wal file since the heap-memory wal-index was created. If so, the + ** heap-memory wal-index is discarded and WAL_RETRY returned to + ** the caller. */ aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); @@ -2276,7 +2280,9 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ } if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; - /* If nTruncate is non-zero, this is a commit record. */ + /* If nTruncate is non-zero, then a complete transaction has been + ** appended to this wal file. Set rc to WAL_RETRY and break out of + ** the loop. */ if( nTruncate ){ rc = WAL_RETRY; break; diff --git a/test/walro2.test b/test/walro2.test index ee4a341e24..9a6b1a1b5c 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -15,7 +15,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl -set ::testprefix walro +source $testdir/wal_common.tcl +set ::testprefix walro2 # These tests are only going to work on unix. # @@ -120,10 +121,81 @@ do_multiclient_test tn { code3 { db3 close } sql1 { COMMIT } } {} - breakpoint do_test 2.3.3 { sql1 { SELECT * FROM t1 } } {a b c d e f g h i j} + + + #----------------------------------------------------------------------- + # 3.1.*: That a readonly_shm connection can read a database file if both + # the *-wal and *-shm files are zero bytes in size. + # + # 3.2.*: That it flushes the cache if, between transactions on a db with a + # zero byte *-wal file, some other connection modifies the db, then + # does "PRAGMA wal_checkpoint=truncate" to truncate the wal file + # back to zero bytes in size. + # + # 3.3.*: That, if between transactions some other process wraps the wal + # file, the readonly_shm client reruns recovery. + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + do_test 3.1.0 { + list [file exists test.db-wal] [file exists test.db-shm] + } {0 0} + do_test 3.1.1 { + close [open test.db-wal w] + close [open test.db-shm w] + code1 { + sqlite3 db file:test.db?readonly_shm=1 + } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h} + + do_test 3.2.0 { + list [file size test.db-wal] [file size test.db-shm] + } {0 0} + do_test 3.2.1 { + code2 { sqlite3 db2 test.db } + sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate } + code2 { db2 close } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h 1 2} + do_test 3.2.2 { + list [file size test.db-wal] [file size test.db-shm] + } {0 32768} + + do_test 3.3.0 { + code2 { sqlite3 db2 test.db } + sql2 { + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t1 VALUES(9, 10); + } + code2 { db2 close } + code1 { db close } + list [file size test.db-wal] [file size test.db-shm] + } [list [wal_file_size 4 1024] 32768] + do_test 3.3.1 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10} + do_test 3.3.2 { + code2 { sqlite3 db2 test.db } + sql2 { + PRAGMA wal_checkpoint; + DELETE FROM t1; + INSERT INTO t1 VALUES('i', 'ii'); + } + code2 { db2 close } + list [file size test.db-wal] [file size test.db-shm] + } [list [wal_file_size 4 1024] 32768] + do_test 3.3.3 { + sql1 { SELECT * FROM t1 } + } {i ii} + } finish_test From ab548384525574b953fa7cec2a21b44b10d145e9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Nov 2017 19:49:34 +0000 Subject: [PATCH 10/33] Add further test cases for the new code on this branch. And a couple of fixes. FossilOrigin-Name: 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db --- manifest | 14 ++--- manifest.uuid | 2 +- src/wal.c | 20 +++---- test/walro2.test | 140 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 84f5558cad..1cc173595f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\sthe\scode\sadded\son\sthis\sbranch. -D 2017-11-04T21:06:35.734 +C Add\sfurther\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch.\sAnd\sa\scouple\sof\sfixes. +D 2017-11-06T19:49:34.916 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 0b3c6b805fc1cf288a7c63b1ac0f78dcc0ad6a54b5b0d72fb0992b16360e7647 +F src/wal.c 32ee6550804a27c155bdeeddd9bf9bc6ca5331a901c763105bccd0b408049d20 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 -F test/walro2.test 811ab176ab8571c59c2aac889fdacc7cff8d7a1ceb083796540c2886620a599f +F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30 -R 5345a5078e9be9411c860b392d7c8c77 +P a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f +R fb9f39269dd42d0221893523e00b9010 U dan -Z 9992407e85299b3993fc8148a1281ba1 +Z ae1d97ffe1b3072414f0d45a94e66fe3 diff --git a/manifest.uuid b/manifest.uuid index ce4b2816dd..d96dff9523 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f \ No newline at end of file +71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index d8612ec850..017e5e865c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2199,7 +2199,8 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ ** from running recovery. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); if( rc!=SQLITE_OK ){ - return (rc==SQLITE_BUSY ? WAL_RETRY : rc); + if( rc==SQLITE_BUSY ) rc = WAL_RETRY; + goto begin_unlocked_out; } pWal->readLock = 0; @@ -2223,7 +2224,10 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); - if( rc!=SQLITE_OK || (szWalhdr.mxFrame==0) ){ + if( rc!=SQLITE_OK ){ + goto begin_unlocked_out; + } + if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); goto begin_unlocked_out; } @@ -2268,16 +2273,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ /* Read and decode the next log frame. */ rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_SHORT_READ ){ - /* If this branch is taken, some other client has truncated the - ** *-wal file since the call to sqlite3OsFileSize() above. This - ** indicates that a read-write client has connected to the system. - ** So retry opening this read transaction. */ - rc = WAL_RETRY; - } - break; - } + if( rc!=SQLITE_OK ) break; if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; /* If nTruncate is non-zero, then a complete transaction has been diff --git a/test/walro2.test b/test/walro2.test index 9a6b1a1b5c..1c59b70ee6 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -196,6 +196,146 @@ do_multiclient_test tn { sql1 { SELECT * FROM t1 } } {i ii} + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test 4.0 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + } + + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code1 { db close } + } {} + + do_test 4.1.1 { + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { SELECT * FROM t1 } + } {hello world} + + do_test 4.1.2 { + code3 { sqlite3 db3 test.db2 } + sql3 { + INSERT INTO t1 VALUES('!'); + PRAGMA wal_checkpoint = truncate; + } + code3 { db3 close } + } {} + do_test 4.1.3 { + sql2 { SELECT * FROM t1 } + } {hello world !} + + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test 4.2.1 { + code1 { sqlite3 db test.db } + sql1 { + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('!'); + + PRAGMA cache_size = 10; + CREATE TABLE t2(x); + + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 + ) + INSERT INTO t2 SELECT randomblob(500) FROM s; + SELECT count(*) FROM t2; + } + } {500} + do_test 4.2.2 { + file size test.db-wal + } {461152} + do_test 4.2.4 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { + SELECT * FROM t1; + SELECT count(*) FROM t2; + } + } {hello world ! ! 0} + + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test 5.0 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + } + + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code1 { db close } + } {} + + do_test 5.1 { + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { + SELECT * FROM t1; + } + } {hello world ! world hello} + + do_test 5.2 { + code1 { + proc handle_read {op args} { + if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { + set ::res2 [sql2 { SELECT * FROM t1 }] + } + puts "$msg xRead $args" + return "SQLITE_OK" + } + testvfs tvfs -fullshm 1 + + sqlite3 db file:test.db2?vfs=tvfs + db eval { SELECT * FROM sqlite_master } + + tvfs filter xRead + tvfs script handle_read + } + sql1 { + PRAGMA wal_checkpoint = truncate; + } + code1 { set ::res2 } + } {hello world ! world hello} + + do_test 5.3 { + code1 { db close } + code1 { tvfs delete } + } {} + + } finish_test From d24c5b1c9834e0bb0cbbc9f8731ef4a9332c1415 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 09:08:43 +0000 Subject: [PATCH 11/33] Add fault-injection tests for the code on this branch. FossilOrigin-Name: a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 --- manifest | 11 ++++---- manifest.uuid | 2 +- test/walrofault.test | 66 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 test/walrofault.test diff --git a/manifest b/manifest index 1cc173595f..dc34f6b3d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch.\sAnd\sa\scouple\sof\sfixes. -D 2017-11-06T19:49:34.916 +C Add\sfault-injection\stests\sfor\sthe\scode\son\sthis\sbranch. +D 2017-11-07T09:08:43.757 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1528,6 +1528,7 @@ F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd +F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1668,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f -R fb9f39269dd42d0221893523e00b9010 +P 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db +R 2894b2184d60284fd42b25cb64811e1a U dan -Z ae1d97ffe1b3072414f0d45a94e66fe3 +Z 3f08ac70c434ed263499224338c46e3a diff --git a/manifest.uuid b/manifest.uuid index d96dff9523..881ec0ad62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db \ No newline at end of file +a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 \ No newline at end of file diff --git a/test/walrofault.test b/test/walrofault.test new file mode 100644 index 0000000000..b396be218c --- /dev/null +++ b/test/walrofault.test @@ -0,0 +1,66 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix walro2 + +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} + +# And only if the build is WAL-capable. +# +ifcapable !wal { + finish_test + return +} + +db close +sqlite3_shutdown +sqlite3_config_uri 1 +sqlite3 db test.db + +do_execsql_test 1.0 { + CREATE TABLE t1(b); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + PRAGMA cache_size = 10; + BEGIN; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) + INSERT INTO t1(b) SELECT randomblob(800) FROM s; +} {wal} +faultsim_save_and_close + +do_faultsim_test 1 -faults oom* -prep { + catch { db close } + faultsim_restore + sqlite3 db file:test.db?readonly_shm=1 +} -body { + execsql { SELECT * FROM t1 } +} -test { + faultsim_test_result {0 {hello world ! world hello}} +} + + + +finish_test From f12ba66cf8f6158965b22033c51703c0f6e1fde0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 15:43:52 +0000 Subject: [PATCH 12/33] On unix, if the *-shm file cannot be opened for read/write access, open it read-only and proceed as if the readonly_shm=1 URI option were specified. FossilOrigin-Name: ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 25 +++++++++++++------------ test/wal2.test | 2 +- test/walro.test | 6 +++++- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index dc34f6b3d8..d0bf050785 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfault-injection\stests\sfor\sthe\scode\son\sthis\sbranch. -D 2017-11-07T09:08:43.757 +C On\sunix,\sif\sthe\s*-shm\sfile\scannot\sbe\sopened\sfor\sread/write\saccess,\sopen\sit\nread-only\sand\sproceed\sas\sif\sthe\sreadonly_shm=1\sURI\soption\swere\sspecified. +D 2017-11-07T15:43:52.117 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c e376adf6014df7d1a73faaaa6c87e6eb9b299b157a58cccff02fad8abc943fe7 +F src/os_unix.c 7de9280173d58b557c602d2a47c2893e536ef9547fa43df92bce0bc2c0d47b0e F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -1501,7 +1501,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c +F test/wal2.test 2d81ffe2a02d9e5c7447b266f7153716cfcba7aecda5ed832db4544617399e29 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1526,7 +1526,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 -F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552 +F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db -R 2894b2184d60284fd42b25cb64811e1a +P a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 +R 7a4e57959f170e8ae96924256d8168d4 U dan -Z 3f08ac70c434ed263499224338c46e3a +Z fa20e28121f5e7127ed54b089c80ff82 diff --git a/manifest.uuid b/manifest.uuid index 881ec0ad62..6168e89f1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 \ No newline at end of file +ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e90e335cb7..c656215c4a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4369,7 +4369,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ struct unixShmNode *pShmNode; /* The underlying mmapped file */ int rc = SQLITE_OK; /* Result code */ unixInodeInfo *pInode; /* The inode of fd */ - char *zShmFilename; /* Name of the file used for SHM */ + char *zShm; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ @@ -4410,14 +4410,14 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); - zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; + zShm = pShmNode->zFilename = (char*)&pShmNode[1]; #ifdef SQLITE_SHM_DIRECTORY - sqlite3_snprintf(nShmFilename, zShmFilename, + sqlite3_snprintf(nShmFilename, zShm, SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath); - sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); #endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; @@ -4431,15 +4431,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - int openFlags = O_RDWR | O_CREAT; - if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - openFlags = O_RDONLY; - pShmNode->isReadonly = 1; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777)); } - pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; + } + pShmNode->isReadonly = 1; } /* If this process is running as root, make sure that the SHM file diff --git a/test/wal2.test b/test/wal2.test index ddff71aed4..a9cafec66d 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1130,7 +1130,7 @@ if {$::tcl_platform(platform) == "unix"} { foreach {tn db_perm wal_perm shm_perm can_open can_read can_write} { 2 00644 00644 00644 1 1 1 3 00644 00400 00644 1 1 0 - 4 00644 00644 00400 1 0 0 + 4 00644 00644 00400 1 1 0 5 00400 00644 00644 1 1 0 7 00644 00000 00644 1 0 0 diff --git a/test/walro.test b/test/walro.test index 150344e151..cae52db6d3 100644 --- a/test/walro.test +++ b/test/walro.test @@ -139,11 +139,15 @@ do_multiclient_test tn { # Now check that if the readonly_shm option is not supplied, or if it # is set to zero, it is not possible to connect to the database without # read-write access to the shm. + # + # UPDATE: os_unix.c now opens the *-shm file in readonly mode + # automatically. + # do_test 1.3.1 { code1 { db close } code1 { sqlite3 db test.db } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j k l}} # Also test that if the -shm file can be opened for read/write access, # it is not if readonly_shm=1 is present in the URI. From 08ecefc5b1bbcbcb3d04989daf454e8fa3007f82 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 21:15:07 +0000 Subject: [PATCH 13/33] Handle the race condition that may occur if another process connects and then checkpoints and truncates the wal file while a readonly-shm client is building its heap-memory wal-index. FossilOrigin-Name: 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/wal.c | 15 ++++++++------ test/walro2.test | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d0bf050785..bb8c19984a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sunix,\sif\sthe\s*-shm\sfile\scannot\sbe\sopened\sfor\sread/write\saccess,\sopen\sit\nread-only\sand\sproceed\sas\sif\sthe\sreadonly_shm=1\sURI\soption\swere\sspecified. -D 2017-11-07T15:43:52.117 +C Handle\sthe\srace\scondition\sthat\smay\soccur\sif\sanother\sprocess\sconnects\sand\sthen\ncheckpoints\sand\struncates\sthe\swal\sfile\swhile\sa\sreadonly-shm\sclient\sis\sbuilding\nits\sheap-memory\swal-index. +D 2017-11-07T21:15:07.949 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 32ee6550804a27c155bdeeddd9bf9bc6ca5331a901c763105bccd0b408049d20 +F src/wal.c b3cd00a165e63ffa91f803fd678f5fc9b7e82745db79bb6c8c220678d5c7ec2f F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd +F test/walro2.test 2e499d89fa825c9d23b53ed4da8e4dcc7017ea16212d6a4f3aec56d1861eaf8e F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339 -R 7a4e57959f170e8ae96924256d8168d4 +P ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee +R e391f2931828dcd8c6e04450ea4ce5f1 U dan -Z fa20e28121f5e7127ed54b089c80ff82 +Z a1807096877d0d55e25dd1ebf350b50a diff --git a/manifest.uuid b/manifest.uuid index 6168e89f1d..011649cacf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee \ No newline at end of file +5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 017e5e865c..1a8bd74b02 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2071,6 +2071,12 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ return 0; } +/* +** This is the value that walTryBeginRead returns when it needs to +** be retried. +*/ +#define WAL_RETRY (-1) + /* ** Read the wal-index header from the wal-index and into pWal->hdr. ** If the wal-header appears to be corrupt, try to reconstruct the @@ -2149,6 +2155,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( pWal->bUnlocked ){ if( rc!=SQLITE_OK ){ walIndexClose(pWal, 0); + pWal->bUnlocked = 0; + assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; } pWal->exclusiveMode = WAL_NORMAL_MODE; } @@ -2156,12 +2165,6 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ return rc; } -/* -** This is the value that walTryBeginRead returns when it needs to -** be retried. -*/ -#define WAL_RETRY (-1) - /* ** Open an "unlocked" transaction. An unlocked transaction is a read ** transaction used by a read-only client in cases where the *-shm diff --git a/test/walro2.test b/test/walro2.test index 1c59b70ee6..5a8ce023eb 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -335,7 +335,60 @@ do_multiclient_test tn { code1 { tvfs delete } } {} + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + do_test 6.1 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + } + + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-shm test.db2-shm + + code1 { db close } + } {} + + do_test 6.2 { + code1 { + set ::nRem 5 + proc handle_read {op args} { + if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { + incr ::nRem -1 + if {$::nRem==0} { + code2 { sqlite3 db2 test.db2 } + sql2 { PRAGMA wal_checkpoint = truncate } + } + } + return "SQLITE_OK" + } + testvfs tvfs -fullshm 1 + + tvfs filter xRead + tvfs script handle_read + + sqlite3 db file:test.db2?readonly_shm=1&vfs=tvfs + db eval { SELECT * FROM t1 } + } + } {hello world ! world hello} + + do_test 6.3 { + code1 { db close } + code1 { tvfs delete } + } {} } finish_test From 6c9d8f640b5d2397e0f1338ae160f11220146d44 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Nov 2017 21:25:15 +0000 Subject: [PATCH 14/33] Update an assert in wal.c. FossilOrigin-Name: 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bb8c19984a..0f9d93ff2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\srace\scondition\sthat\smay\soccur\sif\sanother\sprocess\sconnects\sand\sthen\ncheckpoints\sand\struncates\sthe\swal\sfile\swhile\sa\sreadonly-shm\sclient\sis\sbuilding\nits\sheap-memory\swal-index. -D 2017-11-07T21:15:07.949 +C Update\san\sassert\sin\swal.c. +D 2017-11-07T21:25:15.188 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c b3cd00a165e63ffa91f803fd678f5fc9b7e82745db79bb6c8c220678d5c7ec2f +F src/wal.c 6227c952d86fb9cdc380ffda2413ca4f32a44309c61d10c55dec7dc7bc135ba2 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee -R e391f2931828dcd8c6e04450ea4ce5f1 +P 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc +R 598ffeddb9515e7bdc3f854533f484ab U dan -Z a1807096877d0d55e25dd1ebf350b50a +Z 7844b4272e27592e14850347bc28da72 diff --git a/manifest.uuid b/manifest.uuid index 011649cacf..4a3a422204 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc \ No newline at end of file +94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 1a8bd74b02..4f8d5eadd9 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2844,8 +2844,8 @@ int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - assert( pWal->minFrame>0 ); - for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ + assert( pWal->bUnlocked || pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; From 7e45e3a5d76678356b88ceef773cdd6824008651 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 17:32:12 +0000 Subject: [PATCH 15/33] Change the name of SQLITE_READONLY_CANTLOCK to SQLITE_READONLY_CANTINIT. FossilOrigin-Name: 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/main.c | 2 +- src/os_unix.c | 6 +++--- src/sqlite.h.in | 2 +- src/wal.c | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 0f9d93ff2b..7b0825f535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\san\sassert\sin\swal.c. -D 2017-11-07T21:25:15.188 +C Change\sthe\sname\sof\sSQLITE_READONLY_CANTLOCK\sto\sSQLITE_READONLY_CANTINIT. +D 2017-11-08T17:32:12.655 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -428,7 +428,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2 -F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c +F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 7de9280173d58b557c602d2a47c2893e536ef9547fa43df92bce0bc2c0d47b0e +F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 @@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a -F src/sqlite.h.in ba9029e71a605bc5f236cd693abeb7920285785f2e1a92313ecf8fb1c9f52a86 +F src/sqlite.h.in 49b42d0376e7ed94fe71a7338ea19b20185897abaf6b87fd79ec7126c9541c24 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 6227c952d86fb9cdc380ffda2413ca4f32a44309c61d10c55dec7dc7bc135ba2 +F src/wal.c b1fef64a242ab30b5fdeeaba49150b420608681673b08833acd17024a179cb1e F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc -R 598ffeddb9515e7bdc3f854533f484ab -U dan -Z 7844b4272e27592e14850347bc28da72 +P 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee +R a029a6995bf9493f5c27e7086cc8ad9a +U drh +Z d606e5b93c29b98849932178fdbf9f8a diff --git a/manifest.uuid b/manifest.uuid index 4a3a422204..3e854635d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee \ No newline at end of file +6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f \ No newline at end of file diff --git a/src/main.c b/src/main.c index 49613f6c74..d090845ae3 100644 --- a/src/main.c +++ b/src/main.c @@ -1314,7 +1314,7 @@ const char *sqlite3ErrName(int rc){ case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; diff --git a/src/os_unix.c b/src/os_unix.c index c656215c4a..c5d9aca2c6 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4278,7 +4278,7 @@ static void unixShmPurge(unixFile *pFd){ ** ** If the DMS cannot be locked because this is a readonly_shm=1 ** connection and no other process already holds a lock, return -** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1. +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ struct flock lock; @@ -4311,7 +4311,7 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ }else if( lock.l_type==F_UNLCK ){ if( pShmNode->isReadonly ){ pShmNode->isUnlocked = 1; - rc = SQLITE_READONLY_CANTLOCK; + rc = SQLITE_READONLY_CANTINIT; }else{ rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ @@ -4450,7 +4450,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); rc = unixLockSharedMemory(pDbFd, pShmNode); - if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err; + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0c0ed25b30..1d083cf06f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -511,7 +511,7 @@ int sqlite3_exec( #define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) -#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) diff --git a/src/wal.c b/src/wal.c index 4f8d5eadd9..11cdeb8440 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2099,7 +2099,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); - if( rc==SQLITE_READONLY_CANTLOCK ){ + if( rc==SQLITE_READONLY_CANTINIT ){ assert( page0==0 && pWal->writeLock==0 ); pWal->bUnlocked = 1; pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; @@ -2219,7 +2219,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ ** the unlocked transaction, as the other client may overwrite wal ** frames that this client is still using. */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); - if( rc!=SQLITE_READONLY_CANTLOCK ){ + if( rc!=SQLITE_READONLY_CANTINIT ){ assert( rc!=SQLITE_OK ); rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); goto begin_unlocked_out; @@ -2509,7 +2509,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } if( mxI==0 ){ assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); From 7316960cec0ea2b411946964f4c02538ff7e3e1c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 17:51:10 +0000 Subject: [PATCH 16/33] Turns out that SQLITE_READONLY_CANTLOCK is an historical name that must be preserved. So make a new SQLITE_READLOCK_CANTINIT name instead. FossilOrigin-Name: 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7b0825f535..3f6afa398d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sSQLITE_READONLY_CANTLOCK\sto\sSQLITE_READONLY_CANTINIT. -D 2017-11-08T17:32:12.655 +C Turns\sout\sthat\sSQLITE_READONLY_CANTLOCK\sis\san\shistorical\sname\sthat\smust\nbe\spreserved.\s\sSo\smake\sa\snew\sSQLITE_READLOCK_CANTINIT\sname\sinstead. +D 2017-11-08T17:51:10.682 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a -F src/sqlite.h.in 49b42d0376e7ed94fe71a7338ea19b20185897abaf6b87fd79ec7126c9541c24 +F src/sqlite.h.in df0f3e18bd7210a562dfa4dc7c5b8f57058dfd0f51d771feb5fcfbe3d6ccae5b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee -R a029a6995bf9493f5c27e7086cc8ad9a +P 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f +R 1af2a9a9afceb6f159be2ed742a0108b U drh -Z d606e5b93c29b98849932178fdbf9f8a +Z 83682bfec24bdede404f3ec79bae717f diff --git a/manifest.uuid b/manifest.uuid index 3e854635d3..70823ee687 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f \ No newline at end of file +04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1d083cf06f..db6e0cbae3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -511,9 +511,10 @@ int sqlite3_exec( #define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) -#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) #define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) #define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) From 9214c1efe831ca60df04f0a06bb6b1cfd5ca6d38 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Nov 2017 19:26:27 +0000 Subject: [PATCH 17/33] Extra comments on the sqlite3OsShmMap() call in walBeginUnlocked(). No changes to code. FossilOrigin-Name: 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3f6afa398d..a22cee602e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Turns\sout\sthat\sSQLITE_READONLY_CANTLOCK\sis\san\shistorical\sname\sthat\smust\nbe\spreserved.\s\sSo\smake\sa\snew\sSQLITE_READLOCK_CANTINIT\sname\sinstead. -D 2017-11-08T17:51:10.682 +C Extra\scomments\son\sthe\ssqlite3OsShmMap()\scall\sin\swalBeginUnlocked().\s\sNo\nchanges\sto\scode. +D 2017-11-08T19:26:27.278 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c b1fef64a242ab30b5fdeeaba49150b420608681673b08833acd17024a179cb1e +F src/wal.c 6903d391b6c224e45910795d66f97151428662c5f459d3b69997209dbcadc6f5 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f -R 1af2a9a9afceb6f159be2ed742a0108b +P 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 +R eba2a72214a6da7d512bda6426cfe5de U drh -Z 83682bfec24bdede404f3ec79bae717f +Z 493ea33d970c37f0e10aaccf35cd6d47 diff --git a/manifest.uuid b/manifest.uuid index 70823ee687..a866afde61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 \ No newline at end of file +033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 11cdeb8440..afd0e3f7b6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2210,6 +2210,14 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ /* Try to map the *-shm file again. If it succeeds this time, then ** a non-readonly_shm connection has already connected to the database. ** In this case, start over with opening the transaction. + ** + ** The *-shm file was opened read-only, so sqlite3OsShmMap() can never + ** return SQLITE_OK here, as that would imply that it had established + ** a read/write mapping. A return of SQLITE_READONLY means success - that + ** a mapping has been established to a shared-memory segment that is actively + ** maintained by a writer. SQLITE_READONLY_CANTINIT means that all + ** all connections to the -shm file are read-only and hence the content + ** of the -shm file might be out-of-date. ** ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint ** from taking place. But it does not prevent the wal from being wrapped @@ -2219,8 +2227,8 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ ** the unlocked transaction, as the other client may overwrite wal ** frames that this client is still using. */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ - assert( rc!=SQLITE_OK ); rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); goto begin_unlocked_out; } From 4ff8431fd161d4d55fed8ba4b88a1c37a4357b76 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 16:30:55 +0000 Subject: [PATCH 18/33] Initial work on porting the changes on this branch to Win32. FossilOrigin-Name: 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 --- manifest | 18 ++-- manifest.uuid | 2 +- src/os_win.c | 201 ++++++++++++++++++++++++++++++++++++------- test/walro2.test | 7 -- test/walrofault.test | 7 -- 5 files changed, 179 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index a22cee602e..dbc8286837 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\scomments\son\sthe\ssqlite3OsShmMap()\scall\sin\swalBeginUnlocked().\s\sNo\nchanges\sto\scode. -D 2017-11-08T19:26:27.278 +C Initial\swork\son\sporting\sthe\schanges\son\sthis\sbranch\sto\sWin32. +D 2017-11-09T16:30:55.245 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6 +F src/os_win.c 47687775641c97743c228f99813fbcffe7d53602da5cdcf0fe52f6810341a46c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1527,8 +1527,8 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 2e499d89fa825c9d23b53ed4da8e4dcc7017ea16212d6a4f3aec56d1861eaf8e -F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d +F test/walro2.test 2c01a3c38ca731df4690cc901e3c0bee21bf9d231fa3d47d81162be10462c40b +F test/walrofault.test f1c9c361a73faab98ce1bb4588333e42f7dd330f2b4987e962aa5fe68e7982de F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14 -R eba2a72214a6da7d512bda6426cfe5de -U drh -Z 493ea33d970c37f0e10aaccf35cd6d47 +P 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 +R a3370fa9345068426fa8edd8924927fc +U mistachkin +Z fbbc6e55aa870fcd83aef8acf0e01375 diff --git a/manifest.uuid b/manifest.uuid index a866afde61..f5c23f13f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 \ No newline at end of file +3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 245a86045b..9bfcfe1c81 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2098,6 +2098,17 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; +/* +** The "winIsLockingError" macro is used to determine if a particular I/O +** error code is due to file locking. It must accept the error code DWORD +** as its only argument and should return non-zero if the error code is due +** to file locking. +*/ +#if !defined(winIsLockingError) +#define winIsLockingError(a) (((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_IO_PENDING)) +#endif + /* ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O ** error code obtained via GetLastError() is eligible to be retried. It @@ -3673,6 +3684,9 @@ struct winShmNode { int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ void *pMap; @@ -3820,6 +3834,116 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } } +/* +** Query the status of the DMS lock for the specified file. Returns +** SQLITE_OK upon success. Upon success, the integer pointed to by +** the pLockType argument will be set to the lock type held by the +** other process, as follows: +** +** WINSHM_UNLCK -- No locks are held on the DMS. +** WINSHM_RDLCK -- A SHARED lock is held on the DMS. +** WINSHM_WRLCK -- An EXCLUSIVE lock is held on the DMS. +*/ +static int winGetShmDmsLockType( + winFile *pFile, /* File handle object */ + int *pLockType /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ +){ +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + OVERLAPPED overlapped; /* The offset for ReadFile/WriteFile. */ +#endif + LPVOID pOverlapped = 0; + sqlite3_int64 offset = WIN_SHM_DMS; + BYTE notUsed1 = 0; + DWORD notUsed2 = 0; + +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + if( winSeekFile(pFile, offset) ){ + return SQLITE_IOERR_SEEK; + } +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + pOverlapped = &overlapped; +#endif + if( !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + if( winIsLockingError(osGetLastError()) ){ + *pLockType = WINSHM_WRLCK; + }else{ + return SQLITE_IOERR_READ; + } + }else{ + if( winIsLockingError(osGetLastError()) ){ + *pLockType = WINSHM_RDLCK; + }else{ + return SQLITE_IOERR_WRITE; + } + } + }else{ + *pLockType = WINSHM_UNLCK; + } + return SQLITE_OK; +} + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int winLockSharedMemory(winFile *pDbFd, winShmNode *pShmNode){ + int lockType; + int rc = SQLITE_OK; + + /* Use ReadFile/WriteFile to determine the locks other processes are + ** holding on the DMS byte. If it indicates that another process is + ** holding a SHARED lock, then this process may also take a SHARED + ** lock and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + if( winGetShmDmsLockType(&pShmNode->hFile, &lockType)!=SQLITE_OK ){ + rc = SQLITE_IOERR_LOCK; + }else if( lockType==WINSHM_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK && winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winLockSharedMemory", pShmNode->zFilename); + } + } + }else if( lockType==WINSHM_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lockType==WINSHM_UNLCK || lockType==WINSHM_RDLCK ); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); + } + + return rc; +} + /* ** Open the shared-memory area associated with database file pDbFd. ** @@ -3828,11 +3952,12 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ ** the file must be truncated to zero length or have its header cleared. */ static int winOpenSharedMemory(winFile *pDbFd){ - struct winShm *p; /* The connection to be opened */ - struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc; /* Result code */ - struct winShmNode *pNew; /* Newly allocated winShmNode */ - int nName; /* Size of zName in bytes */ + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + int rc2 = SQLITE_ERROR; /* winOpen result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ assert( pDbFd->pShm==0 ); /* Not previously opened */ @@ -3878,30 +4003,29 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } - rc = winOpen(pDbFd->pVfs, - pShmNode->zFilename, /* Name of the file (UTF-8) */ - (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - 0); - if( SQLITE_OK!=rc ){ - goto shm_open_err; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + rc2 = winOpen(pDbFd->pVfs, + pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + SQLITE_OPEN_WAL|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, + 0); + } + if( rc2!=SQLITE_OK ){ + rc2 = winOpen(pDbFd->pVfs, + pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY|SQLITE_OPEN_CREATE, + 0); + if( rc2!=SQLITE_OK ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(), + "winOpenShm", pShmNode->zFilename); + goto shm_open_err; + } + pShmNode->isReadonly = 1; } - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); - if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winOpenShm", pDbFd->zPath); - } - } - if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + rc = winLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ @@ -4128,6 +4252,8 @@ static int winShmMap( winFile *pDbFd = (winFile*)fd; winShm *pShm = pDbFd->pShm; winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; int rc = SQLITE_OK; if( !pShm ){ @@ -4138,6 +4264,11 @@ static int winShmMap( pShmNode = pShm->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = winLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ @@ -4184,21 +4315,26 @@ static int winShmMap( } pShmNode->aRegion = apNew; + if( pShmNode->isReadonly ){ + protect = PAGE_READONLY; + flags = FILE_MAP_READ; + } + while( pShmNode->nRegion<=iRegion ){ HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, PAGE_READWRITE, nByte, NULL + NULL, protect, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #endif OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", @@ -4208,11 +4344,11 @@ static int winShmMap( int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; #if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFileFromApp(hMap, flags, iOffset - iOffsetShift, szRegion + iOffsetShift ); #else - pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif @@ -4243,6 +4379,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } diff --git a/test/walro2.test b/test/walro2.test index 5a8ce023eb..4b90618fc6 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -18,13 +18,6 @@ source $testdir/lock_common.tcl source $testdir/wal_common.tcl set ::testprefix walro2 -# These tests are only going to work on unix. -# -if {$::tcl_platform(platform) != "unix"} { - finish_test - return -} - # And only if the build is WAL-capable. # ifcapable !wal { diff --git a/test/walrofault.test b/test/walrofault.test index b396be218c..22d4f96bbc 100644 --- a/test/walrofault.test +++ b/test/walrofault.test @@ -17,13 +17,6 @@ source $testdir/tester.tcl source $testdir/malloc_common.tcl set ::testprefix walro2 -# These tests are only going to work on unix. -# -if {$::tcl_platform(platform) != "unix"} { - finish_test - return -} - # And only if the build is WAL-capable. # ifcapable !wal { From 4a9ff91852c8bc23a7a1157d05a789b1a1211dd4 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 17:29:04 +0000 Subject: [PATCH 19/33] Make it possible to use OSTRACE for multi-process testing. FossilOrigin-Name: 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index dbc8286837..d11e831d0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\swork\son\sporting\sthe\schanges\son\sthis\sbranch\sto\sWin32. -D 2017-11-09T16:30:55.245 +C Make\sit\spossible\sto\suse\sOSTRACE\sfor\smulti-process\stesting. +D 2017-11-09T17:29:04.731 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -459,7 +459,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5 -F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c +F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7 -R a3370fa9345068426fa8edd8924927fc +P 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 +R b588bb75e731a71d38f9b889268c4e35 U mistachkin -Z fbbc6e55aa870fcd83aef8acf0e01375 +Z 3f860c7c58efb09a7e9d6ae32c9bc1c6 diff --git a/manifest.uuid b/manifest.uuid index f5c23f13f2..866cdfe91f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 \ No newline at end of file +0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 9427844e09..ca8d26e4f9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1092,8 +1092,15 @@ void sqlite3DebugPrintf(const char *zFormat, ...){ sqlite3VXPrintf(&acc, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); +#ifdef SQLITE_OS_TRACE_PROC + { + extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf); + SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf)); + } +#else fprintf(stdout,"%s", zBuf); fflush(stdout); +#endif } #endif From 8a6fa5d41c3188c93a0b3e01d1395ab88e384a82 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 17:29:18 +0000 Subject: [PATCH 20/33] Corrections to the Win32 porting changes on this branch. FossilOrigin-Name: 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index d11e831d0c..021c050212 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sit\spossible\sto\suse\sOSTRACE\sfor\smulti-process\stesting. -D 2017-11-09T17:29:04.731 +C Corrections\sto\sthe\sWin32\sporting\schanges\son\sthis\sbranch. +D 2017-11-09T17:29:18.701 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 47687775641c97743c228f99813fbcffe7d53602da5cdcf0fe52f6810341a46c +F src/os_win.c ce0b1da0d41a1cc78e33431c72fb4b53f3247b3884841ca0a988761565488824 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7 -R b588bb75e731a71d38f9b889268c4e35 +P 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 +R 83920a29a292bf4d0aadf8e0cc58f982 U mistachkin -Z 3f860c7c58efb09a7e9d6ae32c9bc1c6 +Z f20b1547624a9a7c878450951251f899 diff --git a/manifest.uuid b/manifest.uuid index 866cdfe91f..f22324c47b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 \ No newline at end of file +0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 9bfcfe1c81..d434cfe927 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3895,7 +3895,7 @@ static int winGetShmDmsLockType( ** connection and no other process already holds a lock, return ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ -static int winLockSharedMemory(winFile *pDbFd, winShmNode *pShmNode){ +static int winLockSharedMemory(winShmNode *pShmNode){ int lockType; int rc = SQLITE_OK; @@ -4014,7 +4014,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc2 = winOpen(pDbFd->pVfs, pShmNode->zFilename, (sqlite3_file*)&pShmNode->hFile, - SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY|SQLITE_OPEN_CREATE, + SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY, 0); if( rc2!=SQLITE_OK ){ rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(), @@ -4024,7 +4024,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->isReadonly = 1; } - rc = winLockSharedMemory(pDbFd, pShmNode); + rc = winLockSharedMemory(pShmNode); if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } @@ -4048,7 +4048,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: @@ -4265,7 +4265,7 @@ static int winShmMap( sqlite3_mutex_enter(pShmNode->mutex); if( pShmNode->isUnlocked ){ - rc = winLockSharedMemory(pDbFd, pShmNode); + rc = winLockSharedMemory(pShmNode); if( rc!=SQLITE_OK ) goto shmpage_out; pShmNode->isUnlocked = 0; } From a5150656494e915b675498f8b6b888a336997bec Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 18:21:51 +0000 Subject: [PATCH 21/33] Corrections to Win32 lock detection for SHM files. FossilOrigin-Name: 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 021c050212..eb409a8a6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Corrections\sto\sthe\sWin32\sporting\schanges\son\sthis\sbranch. -D 2017-11-09T17:29:18.701 +C Corrections\sto\sWin32\slock\sdetection\sfor\sSHM\sfiles. +D 2017-11-09T18:21:51.933 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c ce0b1da0d41a1cc78e33431c72fb4b53f3247b3884841ca0a988761565488824 +F src/os_win.c eac2f14343eaf9ff8c02d7025ce459f64dc1334c1f6739be5b6595aabed32ca2 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4 -R 83920a29a292bf4d0aadf8e0cc58f982 +P 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 +R e599061360fdeb01478e7f6faea2d0b3 U mistachkin -Z f20b1547624a9a7c878450951251f899 +Z 310f1e71824409858101a715db0b42e2 diff --git a/manifest.uuid b/manifest.uuid index f22324c47b..238938df65 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 \ No newline at end of file +3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d434cfe927..34069351df 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2105,7 +2105,8 @@ static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; ** to file locking. */ #if !defined(winIsLockingError) -#define winIsLockingError(a) (((a)==ERROR_LOCK_VIOLATION) || \ +#define winIsLockingError(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ ((a)==ERROR_IO_PENDING)) #endif @@ -3867,14 +3868,16 @@ static int winGetShmDmsLockType( pOverlapped = &overlapped; #endif if( !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + DWORD lastErrno = osGetLastError(); if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - if( winIsLockingError(osGetLastError()) ){ + lastErrno = osGetLastError(); + if( winIsLockingError(lastErrno) ){ *pLockType = WINSHM_WRLCK; }else{ return SQLITE_IOERR_READ; } }else{ - if( winIsLockingError(osGetLastError()) ){ + if( winIsLockingError(lastErrno) ){ *pLockType = WINSHM_RDLCK; }else{ return SQLITE_IOERR_WRITE; From 9f4aad446b5ed8b0006c323f4df2db1e5743bf90 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 18:53:51 +0000 Subject: [PATCH 22/33] Further corrections to read-only SHM file handling on Win32. FossilOrigin-Name: 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 18 +++++++++++------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index eb409a8a6a..0ecda71659 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Corrections\sto\sWin32\slock\sdetection\sfor\sSHM\sfiles. -D 2017-11-09T18:21:51.933 +C Further\scorrections\sto\sread-only\sSHM\sfile\shandling\son\sWin32. +D 2017-11-09T18:53:51.070 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c eac2f14343eaf9ff8c02d7025ce459f64dc1334c1f6739be5b6595aabed32ca2 +F src/os_win.c f55a1ae65702e1762dcc175c1b3b32818bcb4d5faee83d2159adafbac94770c4 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6 -R e599061360fdeb01478e7f6faea2d0b3 +P 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 +R b94c408b8a8461b2b6bed04445c4e88e U mistachkin -Z 310f1e71824409858101a715db0b42e2 +Z 2f572d772e98ad83ecab51d538357fde diff --git a/manifest.uuid b/manifest.uuid index 238938df65..3ece53ff59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 \ No newline at end of file +43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 34069351df..6b0bb3dbd0 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2105,8 +2105,9 @@ static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; ** to file locking. */ #if !defined(winIsLockingError) -#define winIsLockingError(a) (((a)==ERROR_ACCESS_DENIED) || \ +#define winIsLockingError(a) (((a)==NO_ERROR) || \ ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_HANDLE_EOF) || \ ((a)==ERROR_IO_PENDING)) #endif @@ -3847,6 +3848,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ */ static int winGetShmDmsLockType( winFile *pFile, /* File handle object */ + int bReadOnly, /* Non-zero if the SHM was opened read-only */ int *pLockType /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ ){ #if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) @@ -3867,24 +3869,25 @@ static int winGetShmDmsLockType( overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); pOverlapped = &overlapped; #endif - if( !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - DWORD lastErrno = osGetLastError(); + if( bReadOnly || + !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ + DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError(); if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ lastErrno = osGetLastError(); if( winIsLockingError(lastErrno) ){ - *pLockType = WINSHM_WRLCK; + if( pLockType ) *pLockType = WINSHM_WRLCK; }else{ return SQLITE_IOERR_READ; } }else{ if( winIsLockingError(lastErrno) ){ - *pLockType = WINSHM_RDLCK; + if( pLockType ) *pLockType = WINSHM_RDLCK; }else{ return SQLITE_IOERR_WRITE; } } }else{ - *pLockType = WINSHM_UNLCK; + if( pLockType ) *pLockType = WINSHM_UNLCK; } return SQLITE_OK; } @@ -3920,7 +3923,8 @@ static int winLockSharedMemory(winShmNode *pShmNode){ ** process might open and use the *-shm file without truncating it. ** And if the *-shm file has been corrupted by a power failure or ** system crash, the database itself may also become corrupt. */ - if( winGetShmDmsLockType(&pShmNode->hFile, &lockType)!=SQLITE_OK ){ + if( winGetShmDmsLockType(&pShmNode->hFile, pShmNode->isReadonly, + &lockType)!=SQLITE_OK ){ rc = SQLITE_IOERR_LOCK; }else if( lockType==WINSHM_UNLCK ){ if( pShmNode->isReadonly ){ From bcb416a9ff954bbc22bd226615edc335a6989666 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 20:02:44 +0000 Subject: [PATCH 23/33] Get read-only SHM file tests passing on Win32. FossilOrigin-Name: abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 32 +++++++++++++++++++++----------- test/walro2.test | 2 ++ test/walrofault.test | 1 + 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 0ecda71659..05976da356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\scorrections\sto\sread-only\sSHM\sfile\shandling\son\sWin32. -D 2017-11-09T18:53:51.070 +C Get\sread-only\sSHM\sfile\stests\spassing\son\sWin32. +D 2017-11-09T20:02:44.785 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c f55a1ae65702e1762dcc175c1b3b32818bcb4d5faee83d2159adafbac94770c4 +F src/os_win.c cf4099958dcc72a9e36ce161638aa369811012fa1e2592415fdc1fdafdf26c8c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1527,8 +1527,8 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 2c01a3c38ca731df4690cc901e3c0bee21bf9d231fa3d47d81162be10462c40b -F test/walrofault.test f1c9c361a73faab98ce1bb4588333e42f7dd330f2b4987e962aa5fe68e7982de +F test/walro2.test bde4b25b701be452ba1436409d9ee418513f77bf4c11229d558fd14282330e00 +F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746 -R b94c408b8a8461b2b6bed04445c4e88e +P 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce +R b51af4203460b2806284a868a4615565 U mistachkin -Z 2f572d772e98ad83ecab51d538357fde +Z 2f0a43c33871392e0df8028f39e6b334 diff --git a/manifest.uuid b/manifest.uuid index 3ece53ff59..dd49a781d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce \ No newline at end of file +abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6b0bb3dbd0..05ae7e67db 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2099,18 +2099,26 @@ static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; /* -** The "winIsLockingError" macro is used to determine if a particular I/O -** error code is due to file locking. It must accept the error code DWORD -** as its only argument and should return non-zero if the error code is due -** to file locking. +** The "winIsLockConflict" macro is used to determine if a particular I/O +** error code is due to a file locking conflict. It must accept the error +** code DWORD as its only argument. */ -#if !defined(winIsLockingError) -#define winIsLockingError(a) (((a)==NO_ERROR) || \ +#if !defined(winIsLockConflict) +#define winIsLockConflict(a) (((a)==NO_ERROR) || \ ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_HANDLE_EOF) || \ ((a)==ERROR_IO_PENDING)) #endif +/* +** The "winIsLockMissing" macro is used to determine if a particular I/O +** error code is due to being unable to obtain a file lock because all or +** part of the range requested within the file is missing. It must accept +** the error code DWORD as its only argument. +*/ +#if !defined(winIsLockMissing) +#define winIsLockMissing(a) (((a)==ERROR_HANDLE_EOF)) +#endif + /* ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O ** error code obtained via GetLastError() is eligible to be retried. It @@ -3874,13 +3882,15 @@ static int winGetShmDmsLockType( DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError(); if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ lastErrno = osGetLastError(); - if( winIsLockingError(lastErrno) ){ + if( winIsLockConflict(lastErrno) ){ if( pLockType ) *pLockType = WINSHM_WRLCK; + }else if( winIsLockMissing(lastErrno) ){ + if( pLockType ) *pLockType = WINSHM_UNLCK; }else{ return SQLITE_IOERR_READ; } }else{ - if( winIsLockingError(lastErrno) ){ + if( winIsLockConflict(lastErrno) ){ if( pLockType ) *pLockType = WINSHM_RDLCK; }else{ return SQLITE_IOERR_WRITE; @@ -4024,8 +4034,8 @@ static int winOpenSharedMemory(winFile *pDbFd){ SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY, 0); if( rc2!=SQLITE_OK ){ - rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(), - "winOpenShm", pShmNode->zFilename); + rc = winLogError(rc2, osGetLastError(), "winOpenShm", + pShmNode->zFilename); goto shm_open_err; } pShmNode->isReadonly = 1; diff --git a/test/walro2.test b/test/walro2.test index 4b90618fc6..67023de1a2 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -255,6 +255,8 @@ do_multiclient_test tn { file size test.db-wal } {461152} do_test 4.2.4 { + file_control_persist_wal db 1; db close + forcecopy test.db test.db2 forcecopy test.db-wal test.db2-wal forcecopy test.db-shm test.db2-shm diff --git a/test/walrofault.test b/test/walrofault.test index 22d4f96bbc..3e66e2d920 100644 --- a/test/walrofault.test +++ b/test/walrofault.test @@ -42,6 +42,7 @@ do_execsql_test 1.0 { WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) INSERT INTO t1(b) SELECT randomblob(800) FROM s; } {wal} +file_control_persist_wal db 1; db close faultsim_save_and_close do_faultsim_test 1 -faults oom* -prep { From e94187780620d5c1b6ceafe3e90ac15f17fe428b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Nov 2017 20:34:35 +0000 Subject: [PATCH 24/33] Enhance walro2.test to better ensure that readonly_shm clients are not using invalid *-shm files. FossilOrigin-Name: ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c --- manifest | 14 +++--- manifest.uuid | 2 +- test/walro2.test | 110 ++++++++++++++++++++++++----------------------- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 05976da356..e214aee04c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sread-only\sSHM\sfile\stests\spassing\son\sWin32. -D 2017-11-09T20:02:44.785 +C Enhance\swalro2.test\sto\sbetter\sensure\sthat\sreadonly_shm\sclients\sare\snot\susing\ninvalid\s*-shm\sfiles. +D 2017-11-09T20:34:35.406 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test bde4b25b701be452ba1436409d9ee418513f77bf4c11229d558fd14282330e00 +F test/walro2.test 8812e514c968bf4ee317571fafedac43443360ae23edd7d0f4ef1eae0c13e8e8 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce -R b51af4203460b2806284a868a4615565 -U mistachkin -Z 2f0a43c33871392e0df8028f39e6b334 +P abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 +R 43c8def15803418ff2b6211daf889aff +U dan +Z b374f730e5d2822de9c4c678f7c53c0c diff --git a/manifest.uuid b/manifest.uuid index dd49a781d5..d156b1b802 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 \ No newline at end of file +ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c \ No newline at end of file diff --git a/test/walro2.test b/test/walro2.test index 67023de1a2..9d0a6b4a19 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -25,6 +25,22 @@ ifcapable !wal { return } +proc copy_to_test2 {bZeroShm} { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + if {$bZeroShm} { + forcedelete test.db2-shm + set fd [open test.db2-shm w] + seek $fd [expr [file size test.db-shm]-1] + puts -nonewline $fd "\0" + close $fd + } else { + forcecopy test.db-shm test.db2-shm + } +} + +foreach bZeroShm {0 1} { +set TN [expr $bZeroShm+1] do_multiclient_test tn { # Close all connections and delete the database. @@ -45,7 +61,7 @@ do_multiclient_test tn { } } - do_test 1.1 { + do_test $TN.1.1 { code2 { sqlite3 db2 test.db } sql2 { CREATE TABLE t1(x, y); @@ -56,26 +72,24 @@ do_multiclient_test tn { file exists test.db-shm } {1} - do_test 1.2.1 { - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + do_test $TN.1.2.1 { + copy_to_test2 $bZeroShm code1 { sqlite3 db file:test.db2?readonly_shm=1 } sql1 { SELECT * FROM t1 } } {a b c d} - do_test 1.2.2 { + do_test $TN.1.2.2 { sql1 { SELECT * FROM t1 } } {a b c d} - do_test 1.3.1 { + do_test $TN.1.3.1 { code3 { sqlite3 db3 test.db2 } sql3 { SELECT * FROM t1 } } {a b c d} - do_test 1.3.2 { + do_test $TN.1.3.2 { sql1 { SELECT * FROM t1 } } {a b c d} @@ -83,7 +97,7 @@ do_multiclient_test tn { code2 { db2 close } code3 { db3 close } - do_test 2.1 { + do_test $TN.2.1 { code2 { sqlite3 db2 test.db } sql2 { INSERT INTO t1 VALUES('e', 'f'); @@ -92,10 +106,8 @@ do_multiclient_test tn { file exists test.db-shm } {1} - do_test 2.2 { - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + do_test $TN.2.2 { + copy_to_test2 $bZeroShm code1 { sqlite3 db file:test.db2?readonly_shm=1 } @@ -105,16 +117,16 @@ do_multiclient_test tn { } } {a b c d e f g h} - do_test 2.3.1 { + do_test $TN.2.3.1 { code3 { sqlite3 db3 test.db2 } sql3 { SELECT * FROM t1 } } {a b c d e f g h} - do_test 2.3.2 { + do_test $TN.2.3.2 { sql3 { INSERT INTO t1 VALUES('i', 'j') } code3 { db3 close } sql1 { COMMIT } } {} - do_test 2.3.3 { + do_test $TN.2.3.3 { sql1 { SELECT * FROM t1 } } {a b c d e f g h i j} @@ -134,10 +146,10 @@ do_multiclient_test tn { catch { code1 { db close } } catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 3.1.0 { + do_test $TN.3.1.0 { list [file exists test.db-wal] [file exists test.db-shm] } {0 0} - do_test 3.1.1 { + do_test $TN.3.1.1 { close [open test.db-wal w] close [open test.db-shm w] code1 { @@ -146,20 +158,20 @@ do_multiclient_test tn { sql1 { SELECT * FROM t1 } } {a b c d e f g h} - do_test 3.2.0 { + do_test $TN.3.2.0 { list [file size test.db-wal] [file size test.db-shm] } {0 0} - do_test 3.2.1 { + do_test $TN.3.2.1 { code2 { sqlite3 db2 test.db } sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate } code2 { db2 close } sql1 { SELECT * FROM t1 } } {a b c d e f g h 1 2} - do_test 3.2.2 { + do_test $TN.3.2.2 { list [file size test.db-wal] [file size test.db-shm] } {0 32768} - do_test 3.3.0 { + do_test $TN.3.3.0 { code2 { sqlite3 db2 test.db } sql2 { INSERT INTO t1 VALUES(3, 4); @@ -171,11 +183,11 @@ do_multiclient_test tn { code1 { db close } list [file size test.db-wal] [file size test.db-shm] } [list [wal_file_size 4 1024] 32768] - do_test 3.3.1 { + do_test $TN.3.3.1 { code1 { sqlite3 db file:test.db?readonly_shm=1 } sql1 { SELECT * FROM t1 } } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10} - do_test 3.3.2 { + do_test $TN.3.3.2 { code2 { sqlite3 db2 test.db } sql2 { PRAGMA wal_checkpoint; @@ -185,7 +197,7 @@ do_multiclient_test tn { code2 { db2 close } list [file size test.db-wal] [file size test.db-shm] } [list [wal_file_size 4 1024] 32768] - do_test 3.3.3 { + do_test $TN.3.3.3 { sql1 { SELECT * FROM t1 } } {i ii} @@ -196,7 +208,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 4.0 { + do_test $TN.4.0 { code1 { forcedelete test.db } code1 { sqlite3 db test.db } sql1 { @@ -206,19 +218,17 @@ do_multiclient_test tn { INSERT INTO t1 VALUES('world'); } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm - + copy_to_test2 $bZeroShm + code1 { db close } } {} - do_test 4.1.1 { + do_test $TN.4.1.1 { code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } sql2 { SELECT * FROM t1 } } {hello world} - do_test 4.1.2 { + do_test $TN.4.1.2 { code3 { sqlite3 db3 test.db2 } sql3 { INSERT INTO t1 VALUES('!'); @@ -226,7 +236,7 @@ do_multiclient_test tn { } code3 { db3 close } } {} - do_test 4.1.3 { + do_test $TN.4.1.3 { sql2 { SELECT * FROM t1 } } {hello world !} @@ -234,7 +244,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 4.2.1 { + do_test $TN.4.2.1 { code1 { sqlite3 db test.db } sql1 { INSERT INTO t1 VALUES('!'); @@ -251,16 +261,13 @@ do_multiclient_test tn { SELECT count(*) FROM t2; } } {500} - do_test 4.2.2 { + do_test $TN.4.2.2 { file size test.db-wal } {461152} - do_test 4.2.4 { + do_test $TN.4.2.4 { file_control_persist_wal db 1; db close - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm - + copy_to_test2 $bZeroShm code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } sql2 { SELECT * FROM t1; @@ -275,7 +282,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 5.0 { + do_test $TN.5.0 { code1 { forcedelete test.db } code1 { sqlite3 db test.db } sql1 { @@ -288,21 +295,19 @@ do_multiclient_test tn { INSERT INTO t1 VALUES('hello'); } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + copy_to_test2 $bZeroShm code1 { db close } } {} - do_test 5.1 { + do_test $TN.5.1 { code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } sql2 { SELECT * FROM t1; } } {hello world ! world hello} - do_test 5.2 { + do_test $TN.5.2 { code1 { proc handle_read {op args} { if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { @@ -325,7 +330,7 @@ do_multiclient_test tn { code1 { set ::res2 } } {hello world ! world hello} - do_test 5.3 { + do_test $TN.5.3 { code1 { db close } code1 { tvfs delete } } {} @@ -337,7 +342,7 @@ do_multiclient_test tn { catch { code2 { db2 close } } catch { code3 { db3 close } } - do_test 6.1 { + do_test $TN.6.1 { code1 { forcedelete test.db } code1 { sqlite3 db test.db } sql1 { @@ -350,14 +355,12 @@ do_multiclient_test tn { INSERT INTO t1 VALUES('hello'); } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcecopy test.db-shm test.db2-shm + copy_to_test2 $bZeroShm code1 { db close } } {} - do_test 6.2 { + do_test $TN.6.2 { code1 { set ::nRem 5 proc handle_read {op args} { @@ -380,10 +383,11 @@ do_multiclient_test tn { } } {hello world ! world hello} - do_test 6.3 { + do_test $TN.6.3 { code1 { db close } code1 { tvfs delete } } {} } +} ;# foreach bZeroShm finish_test From 95a05aae679bb79bd697ab082fc6e6b50356f2c5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 20:37:37 +0000 Subject: [PATCH 25/33] Add an assert() in the Win32 VFS. FossilOrigin-Name: 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e214aee04c..68233db669 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\swalro2.test\sto\sbetter\sensure\sthat\sreadonly_shm\sclients\sare\snot\susing\ninvalid\s*-shm\sfiles. -D 2017-11-09T20:34:35.406 +C Add\san\sassert()\sin\sthe\sWin32\sVFS. +D 2017-11-09T20:37:37.876 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c cf4099958dcc72a9e36ce161638aa369811012fa1e2592415fdc1fdafdf26c8c +F src/os_win.c 64bc61821f75b37ca213da93aef84557c8730be6e0ca93943223b5e57fe6e5a3 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07 -R 43c8def15803418ff2b6211daf889aff -U dan -Z b374f730e5d2822de9c4c678f7c53c0c +P ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c +R 9f94324b020c3efbc23aa89a0c6b1ac2 +U mistachkin +Z b9496fb62e9a99521b89e7f9003a99d8 diff --git a/manifest.uuid b/manifest.uuid index d156b1b802..da346528d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c \ No newline at end of file +22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 05ae7e67db..aff48e25e9 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3885,6 +3885,7 @@ static int winGetShmDmsLockType( if( winIsLockConflict(lastErrno) ){ if( pLockType ) *pLockType = WINSHM_WRLCK; }else if( winIsLockMissing(lastErrno) ){ + assert( bReadOnly ); if( pLockType ) *pLockType = WINSHM_UNLCK; }else{ return SQLITE_IOERR_READ; From 7b7f224c6242f032e2e1d84e4aeb78ede1cbcae1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 22:23:50 +0000 Subject: [PATCH 26/33] Revise and vastly simplify the Win32 SHM file locking semantics, allowing all new tests to pass. FossilOrigin-Name: d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 --- manifest | 12 ++--- manifest.uuid | 2 +- src/os_win.c | 128 ++++---------------------------------------------- 3 files changed, 15 insertions(+), 127 deletions(-) diff --git a/manifest b/manifest index 68233db669..bf3dc40e6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sin\sthe\sWin32\sVFS. -D 2017-11-09T20:37:37.876 +C Revise\sand\svastly\ssimplify\sthe\sWin32\sSHM\sfile\slocking\ssemantics,\sallowing\sall\snew\stests\sto\spass. +D 2017-11-09T22:23:50.758 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 64bc61821f75b37ca213da93aef84557c8730be6e0ca93943223b5e57fe6e5a3 +F src/os_win.c b40d4f98562048b1d084b71bb08367903b937424427ee7b866902dca7b79cb88 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c -R 9f94324b020c3efbc23aa89a0c6b1ac2 +P 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 +R 0c5d22f8af57ea73c927075712f52c29 U mistachkin -Z b9496fb62e9a99521b89e7f9003a99d8 +Z e19a1ec884226a6dec0cb1fab52ceb52 diff --git a/manifest.uuid b/manifest.uuid index da346528d8..2503e60869 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 \ No newline at end of file +d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index aff48e25e9..975df6154b 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2098,27 +2098,6 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; -/* -** The "winIsLockConflict" macro is used to determine if a particular I/O -** error code is due to a file locking conflict. It must accept the error -** code DWORD as its only argument. -*/ -#if !defined(winIsLockConflict) -#define winIsLockConflict(a) (((a)==NO_ERROR) || \ - ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_IO_PENDING)) -#endif - -/* -** The "winIsLockMissing" macro is used to determine if a particular I/O -** error code is due to being unable to obtain a file lock because all or -** part of the range requested within the file is missing. It must accept -** the error code DWORD as its only argument. -*/ -#if !defined(winIsLockMissing) -#define winIsLockMissing(a) (((a)==ERROR_HANDLE_EOF)) -#endif - /* ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O ** error code obtained via GetLastError() is eligible to be retried. It @@ -3844,65 +3823,6 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } } -/* -** Query the status of the DMS lock for the specified file. Returns -** SQLITE_OK upon success. Upon success, the integer pointed to by -** the pLockType argument will be set to the lock type held by the -** other process, as follows: -** -** WINSHM_UNLCK -- No locks are held on the DMS. -** WINSHM_RDLCK -- A SHARED lock is held on the DMS. -** WINSHM_WRLCK -- An EXCLUSIVE lock is held on the DMS. -*/ -static int winGetShmDmsLockType( - winFile *pFile, /* File handle object */ - int bReadOnly, /* Non-zero if the SHM was opened read-only */ - int *pLockType /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ -){ -#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) - OVERLAPPED overlapped; /* The offset for ReadFile/WriteFile. */ -#endif - LPVOID pOverlapped = 0; - sqlite3_int64 offset = WIN_SHM_DMS; - BYTE notUsed1 = 0; - DWORD notUsed2 = 0; - -#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) - if( winSeekFile(pFile, offset) ){ - return SQLITE_IOERR_SEEK; - } -#else - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(offset & 0xffffffff); - overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); - pOverlapped = &overlapped; -#endif - if( bReadOnly || - !osWriteFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError(); - if( !osReadFile(pFile->h, ¬Used1, 1, ¬Used2, pOverlapped) ){ - lastErrno = osGetLastError(); - if( winIsLockConflict(lastErrno) ){ - if( pLockType ) *pLockType = WINSHM_WRLCK; - }else if( winIsLockMissing(lastErrno) ){ - assert( bReadOnly ); - if( pLockType ) *pLockType = WINSHM_UNLCK; - }else{ - return SQLITE_IOERR_READ; - } - }else{ - if( winIsLockConflict(lastErrno) ){ - if( pLockType ) *pLockType = WINSHM_RDLCK; - }else{ - return SQLITE_IOERR_WRITE; - } - } - }else{ - if( pLockType ) *pLockType = WINSHM_UNLCK; - } - return SQLITE_OK; -} - /* ** The DMS lock has not yet been taken on shm file pShmNode. Attempt to ** take it now. Return SQLITE_OK if successful, or an SQLite error @@ -3913,53 +3833,21 @@ static int winGetShmDmsLockType( ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ static int winLockSharedMemory(winShmNode *pShmNode){ - int lockType; - int rc = SQLITE_OK; - - /* Use ReadFile/WriteFile to determine the locks other processes are - ** holding on the DMS byte. If it indicates that another process is - ** holding a SHARED lock, then this process may also take a SHARED - ** lock and proceed with opening the *-shm file. - ** - ** Or, if no other process is holding any lock, then this process - ** is the first to open it. In this case take an EXCLUSIVE lock on the - ** DMS byte and truncate the *-shm file to zero bytes in size. Then - ** downgrade to a SHARED lock on the DMS byte. - ** - ** If another process is holding an EXCLUSIVE lock on the DMS byte, - ** return SQLITE_BUSY to the caller (it will try again). An earlier - ** version of this code attempted the SHARED lock at this point. But - ** this introduced a subtle race condition: if the process holding - ** EXCLUSIVE failed just before truncating the *-shm file, then this - ** process might open and use the *-shm file without truncating it. - ** And if the *-shm file has been corrupted by a power failure or - ** system crash, the database itself may also become corrupt. */ - if( winGetShmDmsLockType(&pShmNode->hFile, pShmNode->isReadonly, - &lockType)!=SQLITE_OK ){ - rc = SQLITE_IOERR_LOCK; - }else if( lockType==WINSHM_UNLCK ){ + int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK ){ if( pShmNode->isReadonly ){ pShmNode->isUnlocked = 1; - rc = SQLITE_READONLY_CANTINIT; - }else{ winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); - if( rc==SQLITE_OK && winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + return SQLITE_READONLY_CANTINIT; + }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), "winLockSharedMemory", pShmNode->zFilename); - } } - }else if( lockType==WINSHM_WRLCK ){ - rc = SQLITE_BUSY; } - if( rc==SQLITE_OK ){ - assert( lockType==WINSHM_UNLCK || lockType==WINSHM_RDLCK ); - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); - } - - return rc; + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } /* From 10813713baec7d78f700c19e51107a19bb698d98 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 22:25:58 +0000 Subject: [PATCH 27/33] Cleanup superfluous whitespace changes. FossilOrigin-Name: a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index bf3dc40e6b..125e07f68d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\sand\svastly\ssimplify\sthe\sWin32\sSHM\sfile\slocking\ssemantics,\sallowing\sall\snew\stests\sto\spass. -D 2017-11-09T22:23:50.758 +C Cleanup\ssuperfluous\swhitespace\schanges. +D 2017-11-09T22:25:58.319 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c b40d4f98562048b1d084b71bb08367903b937424427ee7b866902dca7b79cb88 +F src/os_win.c 670d296c06f8bfce01887baacf52ec19ec2d682b169a6a0ad8b85a830c2b4db0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281 -R 0c5d22f8af57ea73c927075712f52c29 +P d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 +R d62fe3d1eed9309359251a90225ada34 U mistachkin -Z e19a1ec884226a6dec0cb1fab52ceb52 +Z 112940bd575d10def1c3f99e15024bb0 diff --git a/manifest.uuid b/manifest.uuid index 2503e60869..1dbf5a7d5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 \ No newline at end of file +a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 975df6154b..2ee4ad1acf 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3858,12 +3858,12 @@ static int winLockSharedMemory(winShmNode *pShmNode){ ** the file must be truncated to zero length or have its header cleared. */ static int winOpenSharedMemory(winFile *pDbFd){ - struct winShm *p; /* The connection to be opened */ - winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc = SQLITE_OK; /* Result code */ - int rc2 = SQLITE_ERROR; /* winOpen result code */ - winShmNode *pNew; /* Newly allocated winShmNode */ - int nName; /* Size of zName in bytes */ + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + int rc2 = SQLITE_ERROR; /* winOpen result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ assert( pDbFd->pShm==0 ); /* Not previously opened */ From 0e026f403b9900351cfa11c67f9a4682f2911381 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 9 Nov 2017 23:24:29 +0000 Subject: [PATCH 28/33] Avoid superfluous SHM unlock call in the Win32 VFS. FossilOrigin-Name: 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 125e07f68d..453bed54e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanup\ssuperfluous\swhitespace\schanges. -D 2017-11-09T22:25:58.319 +C Avoid\ssuperfluous\sSHM\sunlock\scall\sin\sthe\sWin32\sVFS. +D 2017-11-09T23:24:29.716 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767 -F src/os_win.c 670d296c06f8bfce01887baacf52ec19ec2d682b169a6a0ad8b85a830c2b4db0 +F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0 -R d62fe3d1eed9309359251a90225ada34 +P a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed +R 9d6be609689f3743af6adbbc5cf401b2 U mistachkin -Z 112940bd575d10def1c3f99e15024bb0 +Z 39f42e393c5183ce2049496b416ae681 diff --git a/manifest.uuid b/manifest.uuid index 1dbf5a7d5c..ba222182ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed \ No newline at end of file +5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2ee4ad1acf..086bbf90aa 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3834,6 +3834,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ */ static int winLockSharedMemory(winShmNode *pShmNode){ int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK ){ if( pShmNode->isReadonly ){ pShmNode->isUnlocked = 1; @@ -3846,7 +3847,10 @@ static int winLockSharedMemory(winShmNode *pShmNode){ } } - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + } + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } From 85bc6df2f197c3f5f040b8c17c567b1cf5b20e12 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Nov 2017 20:00:50 +0000 Subject: [PATCH 29/33] Improved comments and variable names in the read-only WAL logic. FossilOrigin-Name: d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 --- manifest | 14 +++--- manifest.uuid | 2 +- src/wal.c | 131 ++++++++++++++++++++++++++++++-------------------- 3 files changed, 87 insertions(+), 60 deletions(-) diff --git a/manifest b/manifest index 453bed54e5..c2392bfdac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssuperfluous\sSHM\sunlock\scall\sin\sthe\sWin32\sVFS. -D 2017-11-09T23:24:29.716 +C Improved\scomments\sand\svariable\snames\sin\sthe\sread-only\sWAL\slogic. +D 2017-11-10T20:00:50.239 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 6903d391b6c224e45910795d66f97151428662c5f459d3b69997209dbcadc6f5 +F src/wal.c 47f8a4493e077b312399b62fd36fd4e8b70535dcd8d9a5caf18b8be019fcf420 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed -R 9d6be609689f3743af6adbbc5cf401b2 -U mistachkin -Z 39f42e393c5183ce2049496b416ae681 +P 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 +R 4efdec1d3561b29c7744e3cfcf4d5f7b +U drh +Z df395ed2f6beb277e98c44e2de48326a diff --git a/manifest.uuid b/manifest.uuid index ba222182ca..354097e8fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 \ No newline at end of file +d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index afd0e3f7b6..aff0e97fa2 100644 --- a/src/wal.c +++ b/src/wal.c @@ -455,7 +455,7 @@ struct Wal { u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ - u8 bUnlocked; + u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ @@ -1271,7 +1271,7 @@ recovery_error: ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bUnlocked ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){ int i; for(i=0; inWiData; i++){ sqlite3_free((void *)pWal->apWiData[i]); @@ -2099,14 +2099,22 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); - if( rc==SQLITE_READONLY_CANTINIT ){ - assert( page0==0 && pWal->writeLock==0 ); - pWal->bUnlocked = 1; - pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; - *pChanged = 1; - }else if( rc!=SQLITE_OK ){ - return rc; + assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */ + if( rc==SQLITE_READONLY_CANTINIT ){ + /* The SQLITE_READONLY_CANTINIT return means that the shared-memory + ** was openable but is not writable, and this thread is unable to + ** confirm that another write-capable connection has the shared-memory + ** open, and hence the content of the shared-memory is unreliable, + ** since the shared-memory might be inconsistent with the WAL file + ** and there is no writer on hand to fix it. */ + assert( page0==0 && pWal->writeLock==0 ); + pWal->bShmUnreliable = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else{ + return rc; /* Any other non-OK return is just an error */ + } }; assert( page0 || pWal->writeLock==0 ); @@ -2122,7 +2130,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ */ assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->bUnlocked==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ + if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; @@ -2152,10 +2160,10 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; } - if( pWal->bUnlocked ){ + if( pWal->bShmUnreliable ){ if( rc!=SQLITE_OK ){ walIndexClose(pWal, 0); - pWal->bUnlocked = 0; + pWal->bShmUnreliable = 0; assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; } @@ -2166,11 +2174,21 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ } /* -** Open an "unlocked" transaction. An unlocked transaction is a read -** transaction used by a read-only client in cases where the *-shm -** file cannot be mapped and its contents cannot be trusted. It is -** assumed that the *-wal file has been read and that a wal-index -** constructed in heap memory is currently available in Wal.apWiData[]. +** Open a transaction in a connection where the shared-memory is read-only +** and where we cannot verify that there is a separate write-capable connection +** on hand to keep the shared-memory up-to-date with the WAL file. +** +** This can happen, for example, when the shared-memory is implemented by +** memory-mapping a *-shm file, where a prior writer has shut down and +** left the *-shm file on disk, and now the present connection is trying +** to use that database but lacks write permission on the *-shm file. +** Other scenarios are also possible, depending on the VFS implementation. +** +** Precondition: +** +** The *-wal file has been read and an appropriate wal-index has been +** constructed in pWal->apWiData[] using heap memory instead of shared +** memory. ** ** If this function returns SQLITE_OK, then the read transaction has ** been successfully opened. In this case output variable (*pChanged) @@ -2182,7 +2200,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** ** If an error occurs, an SQLite error code is returned. */ -static int walBeginUnlocked(Wal *pWal, int *pChanged){ +static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ i64 szWal; /* Size of wal file on disk in bytes */ i64 iOffset; /* Current offset when reading wal file */ u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ @@ -2193,50 +2211,59 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ int rc; /* Return code */ u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ - assert( pWal->bUnlocked ); + assert( pWal->bShmUnreliable ); assert( pWal->readOnly & WAL_SHM_RDONLY ); assert( pWal->nWiData>0 && pWal->apWiData[0] ); /* Take WAL_READ_LOCK(0). This has the effect of preventing any - ** live clients from running a checkpoint, but does not stop them + ** writers from running a checkpoint, but does not stop them ** from running recovery. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); if( rc!=SQLITE_OK ){ if( rc==SQLITE_BUSY ) rc = WAL_RETRY; - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } pWal->readLock = 0; - /* Try to map the *-shm file again. If it succeeds this time, then - ** a non-readonly_shm connection has already connected to the database. - ** In this case, start over with opening the transaction. + /* Check to see if a separate writer has attached to the shared-memory area, + ** thus making the shared-memory "reliable" again. Do this by invoking + ** the xShmMap() routine of the VFS and looking to see if the return + ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. ** - ** The *-shm file was opened read-only, so sqlite3OsShmMap() can never - ** return SQLITE_OK here, as that would imply that it had established - ** a read/write mapping. A return of SQLITE_READONLY means success - that - ** a mapping has been established to a shared-memory segment that is actively - ** maintained by a writer. SQLITE_READONLY_CANTINIT means that all - ** all connections to the -shm file are read-only and hence the content - ** of the -shm file might be out-of-date. - ** - ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint - ** from taking place. But it does not prevent the wal from being wrapped - ** if a checkpoint has already taken place. This means that if another - ** client is connected at this point, it may have already checkpointed - ** the entire wal. In that case it would not be safe to continue with - ** the unlocked transaction, as the other client may overwrite wal - ** frames that this client is still using. */ + ** Once sqlite3OsShmMap() has been called for a file and has returned + ** any SQLITE_READONLY value, it must SQLITE_READONLY or + ** SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** until sqlite3OsShmUnmap() has been called. This is a requirement + ** on the VFS implementation. + ** + ** If the shared-memory is now "reliable" return WAL_RETRY, which will + ** cause the heap-memory WAL-index to be discarded and the actual + ** shared memory to be used in its place. + */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } + /* Reach this point only if the real shared-memory is still unreliable. + ** Assume the in-memory WAL-index substitute is correct and load it + ** into pWal->hdr. + */ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint + ** from taking place. But it does not prevent the wal from being wrapped + ** if a checkpoint has already taken place. This means that if another + ** client is connected at this point, it may have already checkpointed + ** the entire wal. In that case it would not be safe to continue with + ** the this transaction, as the other client may overwrite wal + ** frames that this client is still using. + */ rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); if( rc!=SQLITE_OK ){ - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } /* Check the salt keys at the start of the wal file still match. */ rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); if( rc!=SQLITE_OK ){ - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ rc = WAL_RETRY; - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } /* Allocate a buffer to read frames into */ @@ -2265,7 +2292,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ aFrame = (u8 *)sqlite3_malloc64(szFrame); if( aFrame==0 ){ rc = SQLITE_NOMEM_BKPT; - goto begin_unlocked_out; + goto begin_unreliable_shm_out; } aData = &aFrame[WAL_FRAME_HDRSIZE]; @@ -2298,7 +2325,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; - begin_unlocked_out: + begin_unreliable_shm_out: sqlite3_free(aFrame); if( rc!=SQLITE_OK ){ int i; @@ -2306,7 +2333,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){ sqlite3_free((void*)pWal->apWiData[i]); pWal->apWiData[i] = 0; } - pWal->bUnlocked = 0; + pWal->bShmUnreliable = 0; sqlite3WalEndReadTransaction(pWal); *pChanged = 1; } @@ -2402,7 +2429,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( !useWal ){ assert( rc==SQLITE_OK ); - if( pWal->bUnlocked==0 ){ + if( pWal->bShmUnreliable==0 ){ rc = walIndexReadHdr(pWal, pChanged); } if( rc==SQLITE_BUSY ){ @@ -2433,8 +2460,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( rc!=SQLITE_OK ){ return rc; } - else if( pWal->bUnlocked ){ - return walBeginUnlocked(pWal, pChanged); + else if( pWal->bShmUnreliable ){ + return walBeginShmUnreliable(pWal, pChanged); } } @@ -2789,7 +2816,7 @@ int sqlite3WalFindFrame( ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || (pWal->readLock==0 && pWal->bUnlocked==0) ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ *piRead = 0; return SQLITE_OK; } @@ -2852,7 +2879,7 @@ int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - assert( pWal->bUnlocked || pWal->minFrame>0 ); + assert( pWal->bShmUnreliable || pWal->minFrame>0 ); for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; From 870655bb9e7ef261995c2f3dc6fa5f6c9b115997 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Nov 2017 13:30:44 +0000 Subject: [PATCH 30/33] Further comment improvements in wal.c. No code changes. FossilOrigin-Name: 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 35 ++++++++++++++++++++--------------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index c2392bfdac..e067bdd554 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\sand\svariable\snames\sin\sthe\sread-only\sWAL\slogic. -D 2017-11-10T20:00:50.239 +C Further\scomment\simprovements\sin\swal.c.\s\sNo\scode\schanges. +D 2017-11-11T13:30:44.595 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 47f8a4493e077b312399b62fd36fd4e8b70535dcd8d9a5caf18b8be019fcf420 +F src/wal.c 0e19d4fbb52085697509f9407176e732d440f372f49ca08510a65d1aa976bd07 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633 -R 4efdec1d3561b29c7744e3cfcf4d5f7b +P d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 +R cc050d490699d35569ec56a8515bac08 U drh -Z df395ed2f6beb277e98c44e2de48326a +Z 1e870a391397f9beedb8e100937bac44 diff --git a/manifest.uuid b/manifest.uuid index 354097e8fa..3a4ec03265 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 \ No newline at end of file +346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index aff0e97fa2..87fbda12d6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2230,16 +2230,23 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ ** the xShmMap() routine of the VFS and looking to see if the return ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. ** - ** Once sqlite3OsShmMap() has been called for a file and has returned - ** any SQLITE_READONLY value, it must SQLITE_READONLY or - ** SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, - ** until sqlite3OsShmUnmap() has been called. This is a requirement - ** on the VFS implementation. - ** ** If the shared-memory is now "reliable" return WAL_RETRY, which will ** cause the heap-memory WAL-index to be discarded and the actual ** shared memory to be used in its place. - */ + ** + ** This step is important because, even though this connection is holding + ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might + ** have already checkpointed the WAL file and, while the current + ** is active, wrap the WAL and start overwriting frames that this + ** process wants to use. + ** + ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has + ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY + ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** even if some external agent does a "chmod" to make the shared-memory + ** writable by us, until sqlite3OsShmUnmap() has been called. + ** This is a requirement on the VFS implementation. + */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ @@ -2247,19 +2254,14 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ goto begin_unreliable_shm_out; } - /* Reach this point only if the real shared-memory is still unreliable. + /* We reach this point only if the real shared-memory is still unreliable. ** Assume the in-memory WAL-index substitute is correct and load it ** into pWal->hdr. */ memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); - /* The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint - ** from taking place. But it does not prevent the wal from being wrapped - ** if a checkpoint has already taken place. This means that if another - ** client is connected at this point, it may have already checkpointed - ** the entire wal. In that case it would not be safe to continue with - ** the this transaction, as the other client may overwrite wal - ** frames that this client is still using. + /* Make sure some writer hasn't come in and changed the WAL file out + ** from under us, then disconnected, while we were not looking. */ rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); if( rc!=SQLITE_OK ){ @@ -2283,6 +2285,9 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ goto begin_unreliable_shm_out; } if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + /* Some writer has wrapped the WAL file while we were not looking. + ** Return WAL_RETRY which will cause the in-memory WAL-index to be + ** rebuilt. */ rc = WAL_RETRY; goto begin_unreliable_shm_out; } From c05a063c680af17386b221b977bc4eda1cf56e2b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Nov 2017 20:11:01 +0000 Subject: [PATCH 31/33] In wal.c: improved comments, new assert() and testcase() macros, and replace some magic numbers with appropriate symbolic constants. FossilOrigin-Name: 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 33 +++++++++++++++++++++++---------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index e067bdd554..2fbd04081b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\scomment\simprovements\sin\swal.c.\s\sNo\scode\schanges. -D 2017-11-11T13:30:44.595 +C In\swal.c:\simproved\scomments,\snew\sassert()\sand\stestcase()\smacros,\sand\nreplace\ssome\smagic\snumbers\swith\sappropriate\ssymbolic\sconstants. +D 2017-11-11T20:11:01.646 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 0e19d4fbb52085697509f9407176e732d440f372f49ca08510a65d1aa976bd07 +F src/wal.c 213ddce034c354e640294e77b28097db2a7e10e16c12f6a1158bd7fbcc8e730c F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5 -R cc050d490699d35569ec56a8515bac08 +P 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e +R cc8d47b78b14e89aa0bda73b44dcaa29 U drh -Z 1e870a391397f9beedb8e100937bac44 +Z f012da86d6540243e580ab3866be51e7 diff --git a/manifest.uuid b/manifest.uuid index 3a4ec03265..cbdfc58fda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e \ No newline at end of file +13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 87fbda12d6..6fddf39076 100644 --- a/src/wal.c +++ b/src/wal.c @@ -545,6 +545,11 @@ struct WalIterator { ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are ** numbered from zero. ** +** If the wal-index is currently smaller the iPage pages then the size +** of the wal-index might be increased, but only if it is safe to do +** so. It is safe to enlarge the wal-index if pWal->writeLock is true +** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE. +** ** If this call is successful, *ppPage is set to point to the wal-index ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, ** then an SQLite error code is returned and *ppPage is set to 0. @@ -576,6 +581,8 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); + assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; if( rc==SQLITE_READONLY ){ @@ -2108,15 +2115,21 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** open, and hence the content of the shared-memory is unreliable, ** since the shared-memory might be inconsistent with the WAL file ** and there is no writer on hand to fix it. */ - assert( page0==0 && pWal->writeLock==0 ); + assert( page0==0 ); + assert( pWal->writeLock==0 ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); pWal->bShmUnreliable = 1; pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; *pChanged = 1; }else{ return rc; /* Any other non-OK return is just an error */ } - }; - assert( page0 || pWal->writeLock==0 ); + }else{ + /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock + ** is zero, which prevents the SHM from growing */ + testcase( page0!=0 ); + } + assert( page0!=0 || pWal->writeLock==0 ); /* If the first page of the wal-index has been mapped, try to read the ** wal-index header immediately, without holding any lock. This usually @@ -3661,24 +3674,24 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ - if( pWal->exclusiveMode ){ - pWal->exclusiveMode = 0; + if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_NORMAL_MODE; if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; }else{ /* Already in locking_mode=NORMAL */ rc = 0; } }else if( op>0 ){ - assert( pWal->exclusiveMode==0 ); + assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); assert( pWal->readLock>=0 ); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; } return rc; } From 2e9b0923aca34aa5d18a3be65f7e082c45758cc3 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Nov 2017 05:51:37 +0000 Subject: [PATCH 32/33] Remove some branches in walTryBeginRead() that were added by check-in [ce5d13c2de] but became unreachable with the addition of logic in check-in [18b26843] that enabled read-only clients to parse the WAL file into a heap-memory WAL-index, thus guaranteeing that the WAL-index header is always available. FossilOrigin-Name: 9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 13 +++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2fbd04081b..fc71ea63a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swal.c:\simproved\scomments,\snew\sassert()\sand\stestcase()\smacros,\sand\nreplace\ssome\smagic\snumbers\swith\sappropriate\ssymbolic\sconstants. -D 2017-11-11T20:11:01.646 +C Remove\ssome\sbranches\sin\swalTryBeginRead()\sthat\swere\s\nadded\sby\scheck-in\s[ce5d13c2de]\sbut\sbecame\sunreachable\swith\sthe\saddition\nof\slogic\sin\scheck-in\s[18b26843]\sthat\senabled\sread-only\sclients\sto\sparse\nthe\sWAL\sfile\sinto\sa\sheap-memory\sWAL-index,\sthus\sguaranteeing\sthat\sthe\s\nWAL-index\sheader\sis\salways\savailable. +D 2017-11-13T05:51:37.627 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 213ddce034c354e640294e77b28097db2a7e10e16c12f6a1158bd7fbcc8e730c +F src/wal.c 31ae1a82afccf02e2ff220add690d057dd7e25d9d7ec0b53fe57f2d62a4ba510 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e -R cc8d47b78b14e89aa0bda73b44dcaa29 +P 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b +R be755a48b3ce9c3ee36556946fccdf2c U drh -Z f012da86d6540243e580ab3866be51e7 +Z a296aa345abf23e8a36cfa010f746e55 diff --git a/manifest.uuid b/manifest.uuid index cbdfc58fda..0dbf02629b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b \ No newline at end of file +9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 6fddf39076..382be4e0c8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2418,6 +2418,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ assert( pWal->readLock<0 ); /* Not currently locked */ + /* useWal may only be set for read/write connections */ + assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 ); + /* Take steps to avoid spinning forever if there is a protocol error. ** ** Circumstances that cause a RETRY should only last for the briefest @@ -2484,9 +2487,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } assert( pWal->nWiData>0 ); - assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) ); - pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0; - if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame) + assert( pWal->apWiData[0]!=0 ); + pInfo = walCkptInfo(pWal); + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) @@ -2498,9 +2501,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(0)); walShmBarrier(pWal); if( rc==SQLITE_OK ){ - if( pInfo - && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ /* It is not safe to allow the reader to continue here if frames ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, From 8b17ac1919afa4f2c73869e30f6938db775ea779 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Nov 2017 03:42:52 +0000 Subject: [PATCH 33/33] Improvement to a comment. No changes to code. FossilOrigin-Name: 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fc71ea63a6..50ac558bb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sbranches\sin\swalTryBeginRead()\sthat\swere\s\nadded\sby\scheck-in\s[ce5d13c2de]\sbut\sbecame\sunreachable\swith\sthe\saddition\nof\slogic\sin\scheck-in\s[18b26843]\sthat\senabled\sread-only\sclients\sto\sparse\nthe\sWAL\sfile\sinto\sa\sheap-memory\sWAL-index,\sthus\sguaranteeing\sthat\sthe\s\nWAL-index\sheader\sis\salways\savailable. -D 2017-11-13T05:51:37.627 +C Improvement\sto\sa\scomment.\s\sNo\schanges\sto\scode. +D 2017-11-14T03:42:52.056 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2 @@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 31ae1a82afccf02e2ff220add690d057dd7e25d9d7ec0b53fe57f2d62a4ba510 +F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 @@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b -R be755a48b3ce9c3ee36556946fccdf2c +P 9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 +R 696f834341714d48762ee99cfc8345fc U drh -Z a296aa345abf23e8a36cfa010f746e55 +Z 27bc2b0f1009e1afc7034f11c759006b diff --git a/manifest.uuid b/manifest.uuid index 0dbf02629b..b0b16a61f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3 \ No newline at end of file +486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 382be4e0c8..376151e1a8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2178,6 +2178,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walIndexClose(pWal, 0); pWal->bShmUnreliable = 0; assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + /* walIndexRecover() might have returned SHORT_READ if a concurrent + ** writer truncated the WAL out from under it. If that happens, it + ** indicates that a writer has fixed the SHM file for us, so retry */ if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; } pWal->exclusiveMode = WAL_NORMAL_MODE;