diff --git a/manifest b/manifest index 6536ccb246..9f6a2b2b1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\schanges\sfrom\strunk. -D 2020-07-24T13:49:38.445 +C Merge\senhancements\sfrom\strunk. +D 2020-07-28T17:51:48.981 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -619,7 +619,7 @@ F src/vdbetrace.c fa3bf238002f0bbbdfb66cc8afb0cea284ff9f148d6439bc1f6f2b4c3b7143 F src/vdbevtab.c ee5b4c902fdda2230f9503ac7b84c6d614c91e8f6f4dc1633e2e8dfef8ffb144 F src/vtab.c 5f5fc793092f53bbdfde296c50f563fb7bda58cf48e9cf6a8bdfbc5abd409845 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 231044ecf7d5d78bc705af9dcec6c10ec59e891366362b6be54bb6a0bc7c17db +F src/wal.c 788fc787c608ea9e29a4438f61348b02230dfd538bb0d4ceed704086dc4aec09 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c 3df26a33dc4f54e8771600fb7fdebe1ece0896c2ad68c30ab40b017aa4395049 F src/where.c 2ea911238674e9baaeddf105dddabed92692a01996073c4d4983f9a7efe481f9 @@ -1673,7 +1673,7 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/wal.test 16180bc4becda176428ad02eaea437b4b8f5ae099314de443a4e12b2dcc007a2 -F test/wal2.test 537f59e5c5932e3b45bf3591ae3e48a2601360c2e52821b633e222fe6ebd5b09 +F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1698,7 +1698,7 @@ F test/walmode.test cd6e7cff618eaaa5910ce57c3657aa50110397f86213886a2400afb9bfec F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 -F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84 +F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878 F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 F test/walro2.test 0e79dd15cbdb4f482c01ea248373669c732414a726b357d04846a816afafb768 @@ -1879,7 +1879,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 849b03d8de62251707ee0d31c9f1cd98cbd296c70c1a2c8d7c1658840a930fe4 73fecc688ab5c459245c9bb89432139a083ef378404b4dae3397a7f4e253f72a -R 9214a1b033c89ba9393ee70f0a63232f +P 22e8e6901a119698de831ede6d8b03c4fd6576eaa8686a97a0b8aeea7593688a ee8a108058c304f9b6b02f84f1da01a0b7a3a21992627bcc1f97d42e8d23da69 +R eadae907213e64a4a20abad95f5fd65f U drh -Z d41ecf0a1092c8d49d888e26f4ee5585 +Z c2595a223b4a7c3d570814053566ad4a diff --git a/manifest.uuid b/manifest.uuid index be11d5569e..92cc10c689 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22e8e6901a119698de831ede6d8b03c4fd6576eaa8686a97a0b8aeea7593688a \ No newline at end of file +969c25bb14fbd99ca8523abf0ae78a75a3dde539e3323d105690aef4940041eb \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index c6d4476e7b..0b3250ffaa 100644 --- a/src/wal.c +++ b/src/wal.c @@ -590,7 +590,9 @@ static SQLITE_NOINLINE int walIndexPageRealloc( ); assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); - if( (rc&0xff)==SQLITE_READONLY ){ + if( rc==SQLITE_OK ){ + if( iPage>0 && sqlite3FaultSim(600) ) rc = SQLITE_NOMEM; + }else if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; if( rc==SQLITE_READONLY ){ rc = SQLITE_OK; @@ -1161,12 +1163,6 @@ static int walIndexRecover(Wal *pWal){ assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); - if( rc==SQLITE_OK ){ - rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); - if( rc!=SQLITE_OK ){ - walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); - } - } if( rc ){ return rc; } @@ -1182,15 +1178,16 @@ static int walIndexRecover(Wal *pWal){ if( nSize>WAL_HDRSIZE ){ u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u32 *aPrivate = 0; /* Heap copy of *-shm hash being populated */ u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ int szFrame; /* Number of bytes in buffer aFrame[] */ u8 *aData; /* Pointer to data part of aFrame buffer */ - int iFrame; /* Index of last frame read */ - i64 iOffset; /* Next offset to read from log file */ int szPage; /* Page size according to the log */ u32 magic; /* Magic value read from WAL header */ u32 version; /* Magic value read from WAL header */ int isValid; /* True if this frame is valid */ + int iPg; /* Current 32KB wal-index page */ + int iLastFrame; /* Last frame in wal, based on nSize alone */ /* Read in the WAL header. */ rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); @@ -1237,38 +1234,55 @@ static int walIndexRecover(Wal *pWal){ /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc64(szFrame); + aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); if( !aFrame ){ rc = SQLITE_NOMEM_BKPT; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; + aPrivate = (u32*)&aData[szPage]; /* Read all frames from the log file. */ - iFrame = 0; - for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ - u32 pgno; /* Database page number for frame */ - u32 nTruncate; /* dbsize field from frame header */ + iLastFrame = (nSize - WAL_HDRSIZE) / szFrame; + for(iPg=0; iPg<=walFramePage(iLastFrame); iPg++){ + u32 *aShare; + int iFrame; /* Index of last frame read */ + int iLast = MIN(iLastFrame, HASHTABLE_NPAGE_ONE+iPg*HASHTABLE_NPAGE); + int iFirst = 1 + (iPg==0?0:HASHTABLE_NPAGE_ONE+(iPg-1)*HASHTABLE_NPAGE); + int nHdr, nHdr32; + rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); + if( rc ) break; + pWal->apWiData[iPg] = aPrivate; + + for(iFrame=iFirst; iFrame<=iLast; iFrame++){ + i64 iOffset = walFrameOffset(iFrame, szPage); + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ - /* Read and decode the next log frame. */ - iFrame++; - rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); - if( rc!=SQLITE_OK ) break; - isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); - if( !isValid ) break; - rc = walIndexAppend(pWal, iFrame, pgno); - if( rc!=SQLITE_OK ) break; + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iFrame, pgno); + if( NEVER(rc!=SQLITE_OK) ) break; - /* If nTruncate is non-zero, this is a commit record. */ - if( nTruncate ){ - pWal->hdr.mxFrame = iFrame; - pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; - aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } } + pWal->apWiData[iPg] = aShare; + nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); + nHdr32 = nHdr / sizeof(u32); + memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr); + if( iFrame<=iLast ) break; } sqlite3_free(aFrame); @@ -1283,15 +1297,26 @@ finished: walIndexWriteHdr(pWal); /* Reset the checkpoint-header. This is safe because this thread is - ** currently holding locks that exclude all other readers, writers and - ** checkpointers. + ** currently holding locks that exclude all other writers and + ** checkpointers. Then set the values of read-mark slots 1 through N. */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; pInfo->nBackfillAttempted = pWal->hdr.mxFrame; pInfo->aReadMark[0] = 0; - for(i=1; iaReadMark[i] = READMARK_NOT_USED; - if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; + for(i=1; ihdr.mxFrame ){ + pInfo->aReadMark[i] = pWal->hdr.mxFrame; + }else{ + pInfo->aReadMark[i] = READMARK_NOT_USED; + } + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc!=SQLITE_BUSY ){ + goto recovery_error; + } + } /* If more than one frame was recovered from the log file, report an ** event via sqlite3_log(). This is to help with identifying performance @@ -1309,7 +1334,6 @@ finished: recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); - walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); return rc; } diff --git a/test/wal2.test b/test/wal2.test index 9a56eb41fe..ae6134d8b5 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -122,8 +122,12 @@ do_test wal2-1.1 { } {4 10} set RECOVER [list \ - {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ - {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ + {0 1 lock exclusive} {1 2 lock exclusive} \ + {4 1 lock exclusive} {4 1 unlock exclusive} \ + {5 1 lock exclusive} {5 1 unlock exclusive} \ + {6 1 lock exclusive} {6 1 unlock exclusive} \ + {7 1 lock exclusive} {7 1 unlock exclusive} \ + {1 2 unlock exclusive} {0 1 unlock exclusive} \ ] set READ [list \ {4 1 lock shared} {4 1 unlock shared} \ @@ -394,9 +398,17 @@ set expected_locks [list] lappend expected_locks {1 1 lock exclusive} ;# Lock checkpoint lappend expected_locks {0 1 lock exclusive} ;# Lock writer lappend expected_locks {2 1 lock exclusive} ;# Lock recovery -lappend expected_locks {4 4 lock exclusive} ;# Lock all aReadMark[] +# lappend expected_locks {4 4 lock exclusive} ;# Lock all aReadMark[] +lappend expected_locks {4 1 lock exclusive} ;# Lock aReadMark[1] +lappend expected_locks {4 1 unlock exclusive} ;# Unlock aReadMark[1] +lappend expected_locks {5 1 lock exclusive} +lappend expected_locks {5 1 unlock exclusive} +lappend expected_locks {6 1 lock exclusive} +lappend expected_locks {6 1 unlock exclusive} +lappend expected_locks {7 1 lock exclusive} +lappend expected_locks {7 1 unlock exclusive} lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery -lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] +# lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer lappend expected_locks {3 1 lock exclusive} ;# Lock aReadMark[0] lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0] @@ -625,8 +637,12 @@ do_test wal2-6.4.1 { } {} set RECOVERY { - {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} - {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} + {0 1 lock exclusive} {1 2 lock exclusive} + {4 1 lock exclusive} {4 1 unlock exclusive} + {5 1 lock exclusive} {5 1 unlock exclusive} + {6 1 lock exclusive} {6 1 unlock exclusive} + {7 1 lock exclusive} {7 1 unlock exclusive} + {1 2 unlock exclusive} {0 1 unlock exclusive} } set READMARK0_READ { {3 1 lock shared} {3 1 unlock shared} diff --git a/test/walprotocol.test b/test/walprotocol.test index b1d9e8c01f..a262cdd76d 100644 --- a/test/walprotocol.test +++ b/test/walprotocol.test @@ -52,18 +52,28 @@ do_test 1.1 { set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } - lrange $::locks 0 5 -} [list {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ - {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ + lrange $::locks 0 11 +} [list {0 1 lock exclusive} {1 2 lock exclusive} \ + {4 1 lock exclusive} {4 1 unlock exclusive} \ + {5 1 lock exclusive} {5 1 unlock exclusive} \ + {6 1 lock exclusive} {6 1 unlock exclusive} \ + {7 1 lock exclusive} {7 1 unlock exclusive} \ + {1 2 unlock exclusive} \ + {0 1 unlock exclusive} \ ] do_test 1.2 { db close set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } - lrange $::locks 0 5 -} [list {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ - {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ + lrange $::locks 0 11 +} [list {0 1 lock exclusive} {1 2 lock exclusive} \ + {4 1 lock exclusive} {4 1 unlock exclusive} \ + {5 1 lock exclusive} {5 1 unlock exclusive} \ + {6 1 lock exclusive} {6 1 unlock exclusive} \ + {7 1 lock exclusive} {7 1 unlock exclusive} \ + {1 2 unlock exclusive} \ + {0 1 unlock exclusive} \ ] proc lock_callback {method filename handle lock} { if {$lock == "1 2 lock exclusive"} { return SQLITE_BUSY } @@ -101,7 +111,7 @@ do_test 1.5 { set ::locks [list] sqlite3 db test.db -vfs T catchsql { SELECT * FROM x } -} {1 {locking protocol}} +} {0 z} db close T delete @@ -160,7 +170,7 @@ do_test 2.5 { } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.6 { set ::r -} {1 {locking protocol}} +} {0 {Tehran Qom Markazi Qazvin Gilan Ardabil}} db close db2 close @@ -182,7 +192,7 @@ do_test 2.7 { } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.8 { set ::r -} {1 {locking protocol}} +} {0 {Tehran Qom Markazi Qazvin Gilan Ardabil}} db close db2 close