Stricter enforcement of cell sizes when doing balancing operations on the
btree, in order to catch file corruption sooner. FossilOrigin-Name: 12713f320b2c1def273dd8b7833dddaaad5331aba779d4b1ec9aa949814f38fe
This commit is contained in:
drh
parent
f8f47dd8c5
commit
e3dadac591
14
manifest
14
manifest
@ -1,5 +1,5 @@
|
||||
C Fix\sanother\sfts5\scrash\sthat\scan\soccur\sif\sthe\sdatabase\sis\scorrupted.
|
||||
D 2019-01-23T19:17:05.201
|
||||
C Stricter\senforcement\sof\scell\ssizes\swhen\sdoing\sbalancing\soperations\son\sthe\nbtree,\sin\sorder\sto\scatch\sfile\scorruption\ssooner.
|
||||
D 2019-01-23T19:25:59.893
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F Makefile.in 0e7c107ebcaff26681bc5bcf017557db85aa828d6f7fd652d748b7a78072c298
|
||||
@ -455,7 +455,7 @@ F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df
|
||||
F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab
|
||||
F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
|
||||
F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
|
||||
F src/btree.c 315ccbc0d23ec50c4b65c35bc64ff89f20575ab25a5605a01ae726461ba5dc6f
|
||||
F src/btree.c 60fa67e135492b83c78517502cdd64dd1028da3fb5958cc130d94ba75f140b55
|
||||
F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2
|
||||
F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
|
||||
F src/build.c f07c0b154c23737d1699ee63bba31c8ca8b323e2446b957bc6bfec81a62295fc
|
||||
@ -1802,7 +1802,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 0387cb3add992b2028efe4f2100188d8f9fdfdcb233329857aa4b46a293cfc97
|
||||
R f2949632b0c8dd6250b3de3ff30ae3f1
|
||||
U dan
|
||||
Z 76fed8aa680544a51accd6f3b00ea74a
|
||||
P 44ce8baa47192be03c8f11777904c3c07fa5cc5c97b6d8e81572d380995ac688
|
||||
R 394e54d0902bcab8f75c7f1ab86066f4
|
||||
U drh
|
||||
Z 2303e2a69753b4d3e54ffe2a91200226
|
||||
|
||||
@ -1 +1 @@
|
||||
44ce8baa47192be03c8f11777904c3c07fa5cc5c97b6d8e81572d380995ac688
|
||||
12713f320b2c1def273dd8b7833dddaaad5331aba779d4b1ec9aa949814f38fe
|
||||
204
src/btree.c
204
src/btree.c
@ -6693,9 +6693,72 @@ static void insertCell(
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
** The following parameters determine how many adjacent pages get involved
|
||||
** in a balancing operation. NN is the number of neighbors on either side
|
||||
** of the page that participate in the balancing operation. NB is the
|
||||
** total number of pages that participate, including the target page and
|
||||
** NN neighbors on either side.
|
||||
**
|
||||
** The minimum value of NN is 1 (of course). Increasing NN above 1
|
||||
** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance
|
||||
** in exchange for a larger degradation in INSERT and UPDATE performance.
|
||||
** The value of NN appears to give the best results overall.
|
||||
**
|
||||
** (Later:) The description above makes it seem as if these values are
|
||||
** tunable - as if you could change them and recompile and it would all work.
|
||||
** But that is unlikely. NB has been 3 since the inception of SQLite and
|
||||
** we have never tested any other value.
|
||||
*/
|
||||
#define NN 1 /* Number of neighbors on either side of pPage */
|
||||
#define NB 3 /* (NN*2+1): Total pages involved in the balance */
|
||||
|
||||
/*
|
||||
** A CellArray object contains a cache of pointers and sizes for a
|
||||
** consecutive sequence of cells that might be held on multiple pages.
|
||||
**
|
||||
** The cells in this array are the divider cell or cells from the pParent
|
||||
** page plus up to three child pages. There are a total of nCell cells.
|
||||
**
|
||||
** pRef is a pointer to one of the pages that contributes cells. This is
|
||||
** used to access information such as MemPage.intKey and MemPage.pBt->pageSize
|
||||
** which should be common to all pages that contribute cells to this array.
|
||||
**
|
||||
** apCell[] and szCell[] hold, respectively, pointers to the start of each
|
||||
** cell and the size of each cell. Some of the apCell[] pointers might refer
|
||||
** to overflow cells. In other words, some apCel[] pointers might not point
|
||||
** to content area of the pages.
|
||||
**
|
||||
** A szCell[] of zero means the size of that cell has not yet been computed.
|
||||
**
|
||||
** The cells come from as many as four different pages:
|
||||
**
|
||||
** -----------
|
||||
** | Parent |
|
||||
** -----------
|
||||
** / | \
|
||||
** / | \
|
||||
** --------- --------- ---------
|
||||
** |Child-1| |Child-2| |Child-3|
|
||||
** --------- --------- ---------
|
||||
**
|
||||
** The order of cells is in the array is:
|
||||
**
|
||||
** 1. All cells from Child-1 in order
|
||||
** 2. The first divider cell from Parent
|
||||
** 3. All cells from Child-2 in order
|
||||
** 4. The second divider cell from Parent
|
||||
** 5. All cells from Child-3 in order
|
||||
**
|
||||
** The apEnd[] array holds pointer to the end of page for Child-1, the
|
||||
** Parent, Child-2, the Parent (again), and Child-3. The ixNx[] array
|
||||
** holds the number of cells contained in each of these 5 stages, and
|
||||
** all stages to the left. Hence:
|
||||
** ixNx[0] = Number of cells in Child-1.
|
||||
** ixNx[1] = Number of cells in Child-1 plus 1 for first divider.
|
||||
** ixNx[2] = Number of cells in Child-1 and Child-2 + 1 for 1st divider.
|
||||
** ixNx[3] = Number of cells in Child-1 and Child-2 + both divider cells
|
||||
** ixNx[4] = Total number of cells.
|
||||
*/
|
||||
typedef struct CellArray CellArray;
|
||||
struct CellArray {
|
||||
@ -6703,6 +6766,8 @@ struct CellArray {
|
||||
MemPage *pRef; /* Reference page */
|
||||
u8 **apCell; /* All cells begin balanced */
|
||||
u16 *szCell; /* Local size of all cells in apCell[] */
|
||||
u8 *apEnd[NB*2]; /* MemPage.aDataEnd values */
|
||||
int ixNx[NB*2]; /* Index of at which we move to the next apEnd[] */
|
||||
};
|
||||
|
||||
/*
|
||||
@ -6753,37 +6818,58 @@ static u16 cachedCellSize(CellArray *p, int N){
|
||||
** responsibility of the caller to set it correctly.
|
||||
*/
|
||||
static int rebuildPage(
|
||||
MemPage *pPg, /* Edit this page */
|
||||
CellArray *pCArray, /* Content to be added to page pPg */
|
||||
int iFirst, /* First cell in pCArray to use */
|
||||
int nCell, /* Final number of cells on page */
|
||||
u8 **apCell, /* Array of cells */
|
||||
u16 *szCell /* Array of cell sizes */
|
||||
MemPage *pPg /* The page to be reconstructed */
|
||||
){
|
||||
const int hdr = pPg->hdrOffset; /* Offset of header on pPg */
|
||||
u8 * const aData = pPg->aData; /* Pointer to data for pPg */
|
||||
const int usableSize = pPg->pBt->usableSize;
|
||||
u8 * const pEnd = &aData[usableSize];
|
||||
int i;
|
||||
int i = iFirst; /* Which cell to copy from pCArray*/
|
||||
int j; /* Start of cell content area */
|
||||
int iEnd = i+nCell; /* Loop terminator */
|
||||
u8 *pCellptr = pPg->aCellIdx;
|
||||
u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager);
|
||||
u8 *pData;
|
||||
int k; /* Current slot in pCArray->apEnd[] */
|
||||
u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */
|
||||
|
||||
i = get2byte(&aData[hdr+5]);
|
||||
memcpy(&pTmp[i], &aData[i], usableSize - i);
|
||||
assert( i<iEnd );
|
||||
j = get2byte(&aData[hdr+5]);
|
||||
memcpy(&pTmp[j], &aData[j], usableSize - j);
|
||||
|
||||
for(k=0; pCArray->ixNx[k]<=i && ALWAYS(k<NB*2); k++){}
|
||||
pSrcEnd = pCArray->apEnd[k];
|
||||
|
||||
pData = pEnd;
|
||||
for(i=0; i<nCell; i++){
|
||||
u8 *pCell = apCell[i];
|
||||
while( 1/*exit by break*/ ){
|
||||
u8 *pCell = pCArray->apCell[i];
|
||||
u16 sz = pCArray->szCell[i];
|
||||
assert( sz>0 );
|
||||
if( SQLITE_WITHIN(pCell,aData,pEnd) ){
|
||||
if( ((uptr)(pCell+szCell[i]))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT;
|
||||
if( ((uptr)(pCell+sz))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT;
|
||||
pCell = &pTmp[pCell - aData];
|
||||
}else if( (uptr)(pCell+sz)>(uptr)pSrcEnd
|
||||
&& (uptr)(pCell)<(uptr)pSrcEnd
|
||||
){
|
||||
return SQLITE_CORRUPT_BKPT;
|
||||
}
|
||||
pData -= szCell[i];
|
||||
|
||||
pData -= sz;
|
||||
put2byte(pCellptr, (pData - aData));
|
||||
pCellptr += 2;
|
||||
if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT;
|
||||
memcpy(pData, pCell, szCell[i]);
|
||||
assert( szCell[i]==pPg->xCellSize(pPg, pCell) || CORRUPT_DB );
|
||||
testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) );
|
||||
memcpy(pData, pCell, sz);
|
||||
assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB );
|
||||
testcase( sz!=pPg->xCellSize(pPg,pCell) );
|
||||
i++;
|
||||
if( i>=iEnd ) break;
|
||||
if( pCArray->ixNx[k]<=i ){
|
||||
k++;
|
||||
pSrcEnd = pCArray->apEnd[k];
|
||||
}
|
||||
}
|
||||
|
||||
/* The pPg->nFree field is now set incorrectly. The caller will fix it. */
|
||||
@ -6798,12 +6884,11 @@ static int rebuildPage(
|
||||
}
|
||||
|
||||
/*
|
||||
** Array apCell[] contains nCell pointers to b-tree cells. Array szCell
|
||||
** contains the size in bytes of each such cell. This function attempts to
|
||||
** add the cells stored in the array to page pPg. If it cannot (because
|
||||
** the page needs to be defragmented before the cells will fit), non-zero
|
||||
** is returned. Otherwise, if the cells are added successfully, zero is
|
||||
** returned.
|
||||
** The pCArray objects contains pointers to b-tree cells and the cell sizes.
|
||||
** This function attempts to add the cells stored in the array to page pPg.
|
||||
** If it cannot (because the page needs to be defragmented before the cells
|
||||
** will fit), non-zero is returned. Otherwise, if the cells are added
|
||||
** successfully, zero is returned.
|
||||
**
|
||||
** Argument pCellptr points to the first entry in the cell-pointer array
|
||||
** (part of page pPg) to populate. After cell apCell[0] is written to the
|
||||
@ -6825,18 +6910,23 @@ static int rebuildPage(
|
||||
static int pageInsertArray(
|
||||
MemPage *pPg, /* Page to add cells to */
|
||||
u8 *pBegin, /* End of cell-pointer array */
|
||||
u8 **ppData, /* IN/OUT: Page content -area pointer */
|
||||
u8 **ppData, /* IN/OUT: Page content-area pointer */
|
||||
u8 *pCellptr, /* Pointer to cell-pointer area */
|
||||
int iFirst, /* Index of first cell to add */
|
||||
int nCell, /* Number of cells to add to pPg */
|
||||
CellArray *pCArray /* Array of cells */
|
||||
){
|
||||
int i;
|
||||
u8 *aData = pPg->aData;
|
||||
u8 *pData = *ppData;
|
||||
int iEnd = iFirst + nCell;
|
||||
int i = iFirst; /* Loop counter - cell index to insert */
|
||||
u8 *aData = pPg->aData; /* Complete page */
|
||||
u8 *pData = *ppData; /* Content area. A subset of aData[] */
|
||||
int iEnd = iFirst + nCell; /* End of loop. One past last cell to ins */
|
||||
int k; /* Current slot in pCArray->apEnd[] */
|
||||
u8 *pEnd; /* Maximum extent of cell data */
|
||||
assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */
|
||||
for(i=iFirst; i<iEnd; i++){
|
||||
if( iEnd<=iFirst ) return 0;
|
||||
for(k=0; pCArray->ixNx[k]<=i && ALWAYS(k<NB*2); k++){}
|
||||
pEnd = pCArray->apEnd[k];
|
||||
while( 1 /*Exit by break*/ ){
|
||||
int sz, rc;
|
||||
u8 *pSlot;
|
||||
sz = cachedCellSize(pCArray, i);
|
||||
@ -6851,20 +6941,33 @@ static int pageInsertArray(
|
||||
assert( (pSlot+sz)<=pCArray->apCell[i]
|
||||
|| pSlot>=(pCArray->apCell[i]+sz)
|
||||
|| CORRUPT_DB );
|
||||
if( (uptr)(pCArray->apCell[i]+sz)>(uptr)pEnd
|
||||
&& (uptr)(pCArray->apCell[i])<(uptr)pEnd
|
||||
){
|
||||
assert( CORRUPT_DB );
|
||||
(void)SQLITE_CORRUPT_BKPT;
|
||||
return 1;
|
||||
}
|
||||
memmove(pSlot, pCArray->apCell[i], sz);
|
||||
put2byte(pCellptr, (pSlot - aData));
|
||||
pCellptr += 2;
|
||||
i++;
|
||||
if( i>=iEnd ) break;
|
||||
if( pCArray->ixNx[k]<=i ){
|
||||
k++;
|
||||
pEnd = pCArray->apEnd[k];
|
||||
}
|
||||
}
|
||||
*ppData = pData;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
** Array apCell[] contains nCell pointers to b-tree cells. Array szCell
|
||||
** contains the size in bytes of each such cell. This function adds the
|
||||
** space associated with each cell in the array that is currently stored
|
||||
** within the body of pPg to the pPg free-list. The cell-pointers and other
|
||||
** fields of the page are not updated.
|
||||
** The pCArray object contains pointers to b-tree cells and their sizes.
|
||||
**
|
||||
** This function adds the space associated with each cell in the array
|
||||
** that is currently stored within the body of pPg to the pPg free-list.
|
||||
** The cell-pointers and other fields of the page are not updated.
|
||||
**
|
||||
** This function returns the total number of cells added to the free-list.
|
||||
*/
|
||||
@ -6914,9 +7017,9 @@ static int pageFreeArray(
|
||||
}
|
||||
|
||||
/*
|
||||
** apCell[] and szCell[] contains pointers to and sizes of all cells in the
|
||||
** pages being balanced. The current page, pPg, has pPg->nCell cells starting
|
||||
** with apCell[iOld]. After balancing, this page should hold nNew cells
|
||||
** pCArray contains pointers to and sizes of all cells in the pages being
|
||||
** balanced. The current page, pPg, has pPg->nCell cells starting with
|
||||
** pCArray->apCell[iOld]. After balancing, this page should hold nNew cells
|
||||
** starting at apCell[iNew].
|
||||
**
|
||||
** This routine makes the necessary adjustments to pPg so that it contains
|
||||
@ -7016,24 +7119,9 @@ static int editPage(
|
||||
editpage_fail:
|
||||
/* Unable to edit this page. Rebuild it from scratch instead. */
|
||||
populateCellCache(pCArray, iNew, nNew);
|
||||
return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]);
|
||||
return rebuildPage(pCArray, iNew, nNew, pPg);
|
||||
}
|
||||
|
||||
/*
|
||||
** The following parameters determine how many adjacent pages get involved
|
||||
** in a balancing operation. NN is the number of neighbors on either side
|
||||
** of the page that participate in the balancing operation. NB is the
|
||||
** total number of pages that participate, including the target page and
|
||||
** NN neighbors on either side.
|
||||
**
|
||||
** The minimum value of NN is 1 (of course). Increasing NN above 1
|
||||
** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance
|
||||
** in exchange for a larger degradation in INSERT and UPDATE performance.
|
||||
** The value of NN appears to give the best results overall.
|
||||
*/
|
||||
#define NN 1 /* Number of neighbors on either side of pPage */
|
||||
#define NB (NN*2+1) /* Total pages involved in the balance */
|
||||
|
||||
|
||||
#ifndef SQLITE_OMIT_QUICKBALANCE
|
||||
/*
|
||||
@ -7083,12 +7171,22 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){
|
||||
u8 *pCell = pPage->apOvfl[0];
|
||||
u16 szCell = pPage->xCellSize(pPage, pCell);
|
||||
u8 *pStop;
|
||||
CellArray b;
|
||||
|
||||
assert( sqlite3PagerIswriteable(pNew->pDbPage) );
|
||||
assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) );
|
||||
zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF);
|
||||
rc = rebuildPage(pNew, 1, &pCell, &szCell);
|
||||
if( NEVER(rc) ) return rc;
|
||||
b.nCell = 1;
|
||||
b.pRef = pPage;
|
||||
b.apCell = &pCell;
|
||||
b.szCell = &szCell;
|
||||
b.apEnd[0] = pPage->aDataEnd;
|
||||
b.ixNx[0] = 2;
|
||||
rc = rebuildPage(&b, 0, 1, pNew);
|
||||
if( NEVER(rc) ){
|
||||
releasePage(pNew);
|
||||
return rc;
|
||||
}
|
||||
pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell;
|
||||
|
||||
/* If this is an auto-vacuum database, update the pointer map
|
||||
@ -7568,6 +7666,10 @@ static int balance_nonroot(
|
||||
usableSpace = pBt->usableSize - 12 + leafCorrection;
|
||||
for(i=0; i<nOld; i++){
|
||||
MemPage *p = apOld[i];
|
||||
b.apEnd[i*2] = p->aDataEnd;
|
||||
b.apEnd[i*2+1] = pParent->aDataEnd;
|
||||
b.ixNx[i*2] = cntOld[i];
|
||||
b.ixNx[i*2+1] = cntOld[i]+1;
|
||||
szNew[i] = usableSpace - p->nFree;
|
||||
for(j=0; j<p->nOverflow; j++){
|
||||
szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user