From e6d065a81e6bd7be15c778c3a103d94bf1ad3e0f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Feb 2017 19:58:22 +0000 Subject: [PATCH 1/2] Optimize defragmentPage() in the case where the page contains either one or two free-blocks and a small number of fragmented bytes. FossilOrigin-Name: 202b1c0276aec6b8da64d3277de1ad91c9d62d80 --- manifest | 15 ++++++++----- manifest.uuid | 2 +- src/btree.c | 62 ++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index c20fd94d76..77cbf1aa41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Save\sa\sfew\sbytes\sand\sa\sfew\scycles\sby\ssetting\sVdbe.expmask\sto\szero\sfor\nstatements\sprepared\susing\slegacy\sinterface\ssqlite3_prepare(). -D 2017-02-23T16:30:16.521 +C Optimize\sdefragmentPage()\sin\sthe\scase\swhere\sthe\spage\scontains\seither\sone\sor\ntwo\sfree-blocks\sand\sa\ssmall\snumber\sof\sfragmented\sbytes. +D 2017-02-24T19:58:22.394 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -338,7 +338,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982 +F src/btree.c 1763e0ec3a6cbda48d3a5cb5a7451e46fbc8784d F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f @@ -1557,7 +1557,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d6afd98de3ee8b714dfd6477ead955096f623972 -R d82d78a185c578681c4bad1233af7bd3 +P a8fd705258643863493476f8b42ee981608a339f +R c55510e117c3384aa955b9dc66959921 +T *branch * defragmentpage-opt +T *sym-defragmentpage-opt * +T -sym-trunk * U dan -Z 013ae364526d4d7cd1569b7a9709f5c0 +Z a194b2944e8972170f1e898920de81d2 diff --git a/manifest.uuid b/manifest.uuid index 36ee9348a9..6d120aa8df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8fd705258643863493476f8b42ee981608a339f \ No newline at end of file +202b1c0276aec6b8da64d3277de1ad91c9d62d80 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e78ffef1be..6276a850a0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1317,17 +1317,18 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ /* -** Defragment the page given. All Cells are moved to the -** end of the page and all free space is collected into one -** big FreeBlk that occurs in between the header and cell -** pointer array and the cell content area. +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. ** ** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a ** b-tree page so that there are no freeblocks or fragment bytes, all ** unused bytes are contained in the unallocated space region, and all ** cells are packed tightly at the end of the page. */ -static int defragmentPage(MemPage *pPage){ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ int i; /* Loop counter */ int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ @@ -1342,7 +1343,6 @@ static int defragmentPage(MemPage *pPage){ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt!=0 ); assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); @@ -1354,10 +1354,44 @@ static int defragmentPage(MemPage *pPage){ cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); + iCellFirst = cellOffset + 2*nCell; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( iFree2 ){ + sz2 = get2byte(&data[iFree2+2]); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + } + cbrk = top+sz; + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddrpBt->usableSize; cbrk = usableSize; - iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; + for(i=0; inFree ){ + return SQLITE_CORRUPT_BKPT; + } + + defragment_out: assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; data[hdr+2] = 0; - data[hdr+7] = 0; memset(&data[iCellFirst], 0, cbrk-iCellFirst); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } return SQLITE_OK; } @@ -1537,7 +1573,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); - rc = defragmentPage(pPage); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); assert( gap+nByte<=top ); @@ -7689,7 +7725,7 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0]); + rc = defragmentPage(apNew[0], 0); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) From 3b2ede1f035a86195df4b23c11cd445d6361818e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Feb 2017 16:24:02 +0000 Subject: [PATCH 2/2] Tweak the code on this branch to detect b-tree page corruption in the same cases as the trunk. FossilOrigin-Name: f9863b39d96dce6cb5e49a5f3a445ff3d897a951 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/btree.c | 10 +++++----- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 77cbf1aa41..c6196d9c48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sdefragmentPage()\sin\sthe\scase\swhere\sthe\spage\scontains\seither\sone\sor\ntwo\sfree-blocks\sand\sa\ssmall\snumber\sof\sfragmented\sbytes. -D 2017-02-24T19:58:22.394 +C Tweak\sthe\scode\son\sthis\sbranch\sto\sdetect\sb-tree\spage\scorruption\sin\sthe\ssame\scases\sas\sthe\strunk. +D 2017-02-25T16:24:02.032 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -338,7 +338,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 1763e0ec3a6cbda48d3a5cb5a7451e46fbc8784d +F src/btree.c 03149b0f3fec3c1aa3c50a17e997bb442b867632 F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 F src/build.c 51b473eec465f471d607b54e8dbc00751c3f8a1f @@ -1557,10 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8fd705258643863493476f8b42ee981608a339f -R c55510e117c3384aa955b9dc66959921 -T *branch * defragmentpage-opt -T *sym-defragmentpage-opt * -T -sym-trunk * +P 202b1c0276aec6b8da64d3277de1ad91c9d62d80 +R 7a56121f7b6bd4c4a315ae4d10cd07e6 U dan -Z a194b2944e8972170f1e898920de81d2 +Z 0eab51f3ee55bc511deac00d1a02232c diff --git a/manifest.uuid b/manifest.uuid index 6d120aa8df..190b38aa89 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -202b1c0276aec6b8da64d3277de1ad91c9d62d80 \ No newline at end of file +f9863b39d96dce6cb5e49a5f3a445ff3d897a951 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 6276a850a0..bf65852fe6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1425,11 +1425,11 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ memcpy(&data[cbrk], &src[pc], size); } data[hdr+7] = 0; - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } defragment_out: + if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_BKPT; + } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; @@ -1576,7 +1576,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); - assert( gap+nByte<=top ); + assert( gap+2+nByte<=top ); } @@ -7725,7 +7725,7 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0], 0); + rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2)