From 588400b861b2fa82a591b78a85cf3f460435ba8a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Sep 2014 05:00:25 +0000 Subject: [PATCH 01/51] Reduce the amount of memcpy() required by defragmentPage(). FossilOrigin-Name: 3edab9957cc7bb90b52fd40b02613c2cb03fc166 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/btree.c | 25 ++++++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 131d77b3a9..574bfe2c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. -D 2014-09-26T02:41:05.726 +C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). +D 2014-09-27T05:00:25.096 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e +F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 -R 32be19747fff5e8f2465eed0f224b45d +P 83913515830aa850f9e38406f9422d7e88dcab66 +R 66a1e1f00a844450677737824735607d +T *branch * defrag-opt +T *sym-defrag-opt * +T -sym-trunk * U drh -Z f56b9000203c19d0f3a8172e8374b279 +Z f60a1f2e4650c574e91ad245c0c67dee diff --git a/manifest.uuid b/manifest.uuid index b182c8c316..bf8d17f152 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file +3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 135b40139f..c4832b4ee9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1151,6 +1151,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -1160,15 +1161,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -1187,7 +1186,7 @@ static int defragmentPage(MemPage *pPage){ } #endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = cellSizePtr(pPage, &src[pc]); cbrk -= size; #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) if( cbrk=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -5955,13 +5962,13 @@ static void assemblePage( assert( pPage->nCell==0 ); assert( get2byteNotZero(&data[hdr+5])==nUsable ); - pCellptr = &pPage->aCellIdx[nCell*2]; + pCellptr = pPage->aCellIdx; cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ + for(i=0; i Date: Thu, 9 Oct 2014 19:35:37 +0000 Subject: [PATCH 02/51] Change the balance_nonroot() routine to reduce the amount of memcpy work that takes place. This is a work in progress. FossilOrigin-Name: 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 --- manifest | 21 +-- manifest.uuid | 2 +- src/btree.c | 494 +++++++++++++++++++++++++++----------------------- src/pager.c | 9 + src/pager.h | 2 + 5 files changed, 292 insertions(+), 236 deletions(-) diff --git a/manifest b/manifest index 574bfe2c8d..aeefa53238 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). -D 2014-09-27T05:00:25.096 +C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. +D 2014-10-09T19:35:37.452 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 +F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c caab007743821d96752597c9cfd7351654697b06 -F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 +F src/pager.c 0abcb0904a78d68b96357f360c6b160bcfc2a3e0 +F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 F src/parse.y b98772da2bb5415970085b707203f92569400aa8 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1200,10 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83913515830aa850f9e38406f9422d7e88dcab66 -R 66a1e1f00a844450677737824735607d -T *branch * defrag-opt -T *sym-defrag-opt * -T -sym-trunk * -U drh -Z f60a1f2e4650c574e91ad245c0c67dee +P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 +R 5fca1836b5a4d862df24682ecb47d048 +U dan +Z 8527330c8f275358262176fc962502e2 diff --git a/manifest.uuid b/manifest.uuid index bf8d17f152..bb422fd7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file +29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c4832b4ee9..dbf002ecdb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5977,6 +5977,49 @@ static void assemblePage( pPage->nCell = (u16)nCell; } + +static void rebuildPage( + MemPage *pPg, /* Edit this page */ + int nRemove, /* Cells to remove from start of page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of nCell final cells */ + u16 *szCell /* Array of nCell cell sizes */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); + pData = &aData[usableSize]; + + for(i=0; iaData && pCellnFree = (pData - pCellptr); + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6098,7 +6141,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 0 +#if 1 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6265,7 +6308,6 @@ static int balance_nonroot( int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ @@ -6276,6 +6318,13 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ + int aShiftLeft[NB+2]; + int aShiftRight[NB+2]; + u8 abDone[NB+2]; + Pgno aPgno[NB+2]; + u16 aPgFlags[NB+2]; + + memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -6384,12 +6433,10 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ + + pBt->pageSize; /* aSpace1 */ apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; @@ -6402,8 +6449,8 @@ static int balance_nonroot( /* ** Load pointers to all cells on sibling pages and the divider cells ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied @@ -6419,15 +6466,7 @@ static int balance_nonroot( leafData = apOld[0]->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + MemPage *pOld = apOld[i]; limit = pOld->nCell+pOld->nOverflow; if( pOld->nOverflow>0 ){ @@ -6556,10 +6595,10 @@ static int balance_nonroot( assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); #endif - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* @@ -6582,6 +6621,7 @@ static int balance_nonroot( assert( i>0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; @@ -6595,135 +6635,223 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + } + for(i=0; ipgno); + Pgno iMin = 0; + u16 flags = 0; + for(j=0; jiGt && (iMin==0 || iPgnoi ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + if( apNew[i]->pgno!=iMin ){ + apNew[i]->pDbPage->flags = flags; + sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + apNew[i]->pgno = iMin; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. - */ j = 0; for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + /* At this point, "j" is the apCell[] index of the first cell currently + ** stored on page apNew[i]. Or, if apNew[i] was not one of the original + ** sibling pages, "j" should be set to nCell. Variable iFirst is set + ** to the apCell[] index of the first cell that will appear on the + ** page following this balancing operation. */ + int iFirst = (i==0 ? 0 : cntNew[i-1] + !leafData); /* new first cell */ + assert( inCell + apNew[i]->nOverflow; + aShiftRight[i] = cntNew[i] - j; + assert( i!=nOld-1 || j==nCell ); + if( jnOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( iaData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; - assert( jleaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow page), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); } } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - - j++; - nxDiv++; } } - assert( j==nCell ); + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** 1) If the aShiftLeft[] entry is less than 0, it is not safe to + ** update the page until the page to the left of the current page + ** (apNew[i-1]) has already been updated. + ** + ** 2) If the aShiftRight[] entry is less than 0, it is not safe to + ** update the page until the page to the right of the current page + ** (apNew[i+1]) has already been updated. + ** + ** If neither of the above apply, the page is safe to update. + */ + assert( aShiftRight[nNew-1]>=0 && aShiftLeft[0]==0 ); + for(i=0; i=nNew ? i-nNew : nNew-1-i); + if( abDone[iPg]==0 + && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) + && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + ){ + MemPage *pNew = apNew[iPg]; + int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); + rebuildPage(pNew, + aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, + cntNew[iPg] - iLeft, + &apCell[iLeft], + &szCell[iLeft] + ); + abDone[iPg] = 1; + assert( pNew->nOverflow==0 ); + assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + } + } + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -6746,116 +6874,36 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); -#if 0 + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. diff --git a/src/pager.c b/src/pager.c index 79bfe15f10..e68d147de5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6835,6 +6835,14 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } + +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ + PgHdr *pPg = (PgHdr*)pPage; + assert( pPg->flags & PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) ); + sqlite3PcacheMove(pPg, iNew); +} + #endif /* @@ -7235,4 +7243,5 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index c9ca8553b9..f3a04bbca6 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,6 +188,8 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +void sqlite3PagerRekey(DbPage*, Pgno); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif From 09c6840153697ea6679650b65dfa50bc15cef536 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 11 Oct 2014 20:00:24 +0000 Subject: [PATCH 03/51] Attempt to further reduce memcpy() in balance_nonroot(). FossilOrigin-Name: fec849dcca3aead2bc2d4ecffeda750684d32fb0 --- manifest | 12 ++-- manifest.uuid | 2 +- src/btree.c | 185 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index aeefa53238..7e782aa82c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. -D 2014-10-09T19:35:37.452 +C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). +D 2014-10-11T20:00:24.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 +F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 -R 5fca1836b5a4d862df24682ecb47d048 +P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 +R 3235ac37769ff059d7c714947e8a11dd U dan -Z 8527330c8f275358262176fc962502e2 +Z 17fdfd7faf788bbb79d2ec9941c17025 diff --git a/manifest.uuid b/manifest.uuid index bb422fd7dd..006b3c0008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file +fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dbf002ecdb..13ec912748 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5980,10 +5980,9 @@ static void assemblePage( static void rebuildPage( MemPage *pPg, /* Edit this page */ - int nRemove, /* Cells to remove from start of page */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of nCell final cells */ - u16 *szCell /* Array of nCell cell sizes */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ @@ -6020,6 +6019,129 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static void editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + + if( 1 ){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nFree = pPg->nFree; /* Free bytes on pPg */ + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nShift++; + } + } + nCell -= nShift; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + } + for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nCell--; + } + } + pData = &aData[get2byte(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); + for(i=iNew; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + int sz = szCell[iCell+iNew]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + pData -= sz; + if( pDataaCellIdx[nCell*2]; + for(i=iNew+nCell; i<(iNew+nNew); i++){ + pData -= szCell[i]; + if( pDatanFree = nFree; + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } +#endif + +#if 0 + printf("EDIT\n"); +#endif + + return; + } + + editpage_fail: +#if 0 + printf("REBUILD\n"); +#endif + /* Unable to edit this page. Rebuild it from scratch instead. */ + rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6312,6 +6434,7 @@ static int balance_nonroot( u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ int szNew[NB+2]; /* Combined size of cells place on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ @@ -6487,6 +6610,7 @@ static int balance_nonroot( nCell++; } } + cntOld[i] = nCell; if( i %d@%d", apNew[i]->pgno, + pNew->nCell+pNew->nOverflow, j, + cntNew[i] - iFirst, iFirst + ); + for(iCell=iFirst; iCellaData + || apCell[iCell]>=&pNew->aData[pBt->usableSize] + ){ + nCta += szCell[iCell]; + } + } + nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); + nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); + printf(" cta=%d free=%d\n", nCta, nFree); + if( i==(nNew-1) ){ + printf("-----\n"); + fflush(stdout); + } +#endif + assert( inCell + apNew[i]->nOverflow; @@ -6835,17 +6987,23 @@ static int balance_nonroot( && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) && (aShiftRight[iPg]>=0 || abDone[iPg+1]) ){ - MemPage *pNew = apNew[iPg]; - int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); - rebuildPage(pNew, - aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, - cntNew[iPg] - iLeft, - &apCell[iLeft], - &szCell[iLeft] - ); + int iNew; + int iOld; + int nNewCell; + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnOverflow==0 ); - assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); } } assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); @@ -6869,6 +7027,7 @@ static int balance_nonroot( ** is important if the parent page happens to be page 1 of the database ** image. */ assert( nNew==1 ); + defragmentPage(apNew[0]); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) ); From d7b545bbcb2225a33e8c8c5192ab11416a855999 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Oct 2014 18:03:27 +0000 Subject: [PATCH 04/51] Further work on balance_nonroot(). FossilOrigin-Name: 6594f9b420e2fa642737722ff8521f756ecef227 --- manifest | 12 +-- manifest.uuid | 2 +- src/btree.c | 265 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 180 insertions(+), 99 deletions(-) diff --git a/manifest b/manifest index 7e782aa82c..9f25f390d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). -D 2014-10-11T20:00:24.552 +C Further\swork\son\sbalance_nonroot(). +D 2014-10-13T18:03:27.743 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 +F src/btree.c 63211ca1d4ae867eede39a37901aec4746d904a7 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 -R 3235ac37769ff059d7c714947e8a11dd +P fec849dcca3aead2bc2d4ecffeda750684d32fb0 +R 79e5b5564c934fc68257d82dc897cefd U dan -Z 17fdfd7faf788bbb79d2ec9941c17025 +Z ae5b74eee28666050af6baaa639de56c diff --git a/manifest.uuid b/manifest.uuid index 006b3c0008..fcba485229 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file +6594f9b420e2fa642737722ff8521f756ecef227 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 13ec912748..7c9a7228de 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6009,7 +6009,7 @@ static void rebuildPage( assert( szCell[i]==cellSizePtr(pPg, pCell) ); } - pPg->nFree = (pData - pCellptr); + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ pPg->nCell = nCell; pPg->nOverflow = 0; @@ -6019,6 +6019,106 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static u8 *pageFindSlot(MemPage *pPg, int nByte){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + +static int pageInsertArray( + MemPage *pPg, + u8 *pBegin, + u8 **ppData, + u8 *pCellptr, + int nCell, + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + for(i=0; iaData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + int nRet = 0; + int i; + u8 *pFree = 0; + int szFree = 0; + + for(i=0; iaData && pCellnFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ static void editPage( MemPage *pPg, /* Edit this page */ int iOld, /* Index of first cell currently on page */ @@ -6027,113 +6127,93 @@ static void editPage( u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ - - if( 1 ){ - u8 * const aData = pPg->aData; - const int hdr = pPg->hdrOffset; - u8 *pBegin = &pPg->aCellIdx[nNew * 2]; - int nFree = pPg->nFree; /* Free bytes on pPg */ - int nCell = pPg->nCell; /* Cells stored on pPg */ - u8 *pData; - u8 *pCellptr; - int i; - int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; #ifdef SQLITE_DEBUG - u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); - memcpy(pTmp, aData, pPg->pBt->usableSize); + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); #endif - /* Remove cells from the start and end of the page */ - if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nShift++; - } - } - nCell -= nShift; - memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); - } - for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nCell--; - } - } - pData = &aData[get2byte(&aData[hdr+5])]; - if( pDataaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray( + pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] + ); + } - /* Add cells to the start of the page */ - if( iNewaCellIdx; - memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); - for(i=iNew; iaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nAdd, &apCell[iNew], &szCell[iNew] + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + 1, &apCell[iCell + iNew], &szCell[iCell + iNew] + ) ) goto editpage_fail; } + } - /* Add any overflow cells */ - for(i=0; inOverflow; i++){ - int iCell = (iOld + pPg->aiOvfl[i]) - iNew; - if( iCell>=0 && iCellaCellIdx[iCell * 2]; - int sz = szCell[iCell+iNew]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); - pData -= sz; - if( pDataaCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] + ) ) goto editpage_fail; - /* Append cells to the end of the page */ - pCellptr = &pPg->aCellIdx[nCell*2]; - for(i=iNew+nCell; i<(iNew+nNew); i++){ - pData -= szCell[i]; - if( pDatanCell = nNew; + pPg->nOverflow = 0; - pPg->nFree = nFree; - pPg->nCell = nNew; - pPg->nOverflow = 0; - - put2byte(&aData[hdr+3], pPg->nCell); - put2byte(&aData[hdr+5], pData - aData); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); #ifdef SQLITE_DEBUG - for(i=0; iaCellIdx[i*2]); - if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ - pCell = &pTmp[pCell - aData]; - } - assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } #endif #if 0 - printf("EDIT\n"); +printf("EDIT\n"); #endif - return; - } - + return; editpage_fail: #if 0 printf("REBUILD\n"); @@ -6662,7 +6742,7 @@ static int balance_nonroot( assert( i usableSpace ){ - szNew[k] = subtotal - szCell[i]; + szNew[k] = subtotal - szCell[i] - 2; cntNew[k] = i; if( leafData ){ i--; } subtotal = 0; @@ -7002,6 +7082,7 @@ static int balance_nonroot( editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); abDone[iPg] = 1; + apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } From 8e9ba0c335c5dc263f18f42f5c014233c7ad8976 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Oct 2014 17:27:04 +0000 Subject: [PATCH 05/51] Fix some code duplication issues on this branch. Add minor optimizations to the new code. FossilOrigin-Name: 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba --- manifest | 12 +- manifest.uuid | 2 +- src/btree.c | 295 +++++++++++++++++++++----------------------------- 3 files changed, 128 insertions(+), 181 deletions(-) diff --git a/manifest b/manifest index f6d9403e16..347e58fbc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-10-13T18:09:13.737 +C Fix\ssome\scode\sduplication\sissues\son\sthis\sbranch.\sAdd\sminor\soptimizations\sto\sthe\snew\scode. +D 2014-10-14T17:27:04.829 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3d9d1db1d05724346876cf456a850628a9e75331 +F src/btree.c 3c72c5aa96b1a06ea15da24b82c526e6f14010ab F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6594f9b420e2fa642737722ff8521f756ecef227 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 -R 76183a9b7ddeb938a48604dbc4dd8cd6 +P d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 +R ff1ec1f8aba483920a0100e96e95dccf U dan -Z 504cabb1e1fb0c47450f76fec3d8495b +Z 9c750816a0b42cc370356f0c03af45ae diff --git a/manifest.uuid b/manifest.uuid index 32265a861c..065d26bf44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 \ No newline at end of file +58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7136b45b24..acf466304f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1224,6 +1224,57 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If it does and argument +** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** corruption goes unreported. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -1274,33 +1325,12 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+1==top ); testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( data[hdr+7]>=60 ) goto defragment_page; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + int rc = SQLITE_OK; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( rc ) return rc; + if( pSpace ){ + *pIdx = pSpace - data; + return SQLITE_OK; } } @@ -1309,7 +1339,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ -defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -5933,48 +5962,18 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. */ -static void assemblePage( - MemPage *pPage, /* The page to be assembled */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ -){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ - - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); - - pCellptr = pPage->aCellIdx; - cellbody = nUsable; - for(i=0; inFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; -} - - static void rebuildPage( MemPage *pPg, /* Edit this page */ int nCell, /* Final number of cells on page */ @@ -5992,8 +5991,8 @@ static void rebuildPage( i = get2byte(&aData[hdr+5]); memcpy(&pTmp[i], &aData[i], usableSize - i); - pData = &aData[usableSize]; + pData = pEnd; for(i=0; iaData && pCellhdrOffset; - u8 * const aData = pPg->aData; - int iAddr; - int pc; - int usableSize = pPg->pBt->usableSize; - - for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&aData[iAddr], &aData[pc], 2); - aData[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return 0; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&aData[pc+2], x); - } - return &aData[pc + x]; - } - } - - return 0; -} - +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ static int pageInsertArray( - MemPage *pPg, - u8 *pBegin, - u8 **ppData, - u8 *pCellptr, - int nCell, + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int nCell, /* Number of cells to add to pPg */ u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ int i; u8 *aData = pPg->aData; u8 *pData = *ppData; + const int bFreelist = aData[1] || aData[2]; + assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; inFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. @@ -6206,15 +6205,8 @@ static void editPage( } #endif -#if 0 -printf("EDIT\n"); -#endif - return; editpage_fail: -#if 0 - printf("REBUILD\n"); -#endif /* Unable to edit this page. Rebuild it from scratch instead. */ rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); } @@ -6288,7 +6280,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rebuildPage(pNew, 1, &pCell, &szCell); + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -6518,8 +6511,6 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - int aShiftLeft[NB+2]; - int aShiftRight[NB+2]; u8 abDone[NB+2]; Pgno aPgno[NB+2]; u16 aPgFlags[NB+2]; @@ -6887,49 +6878,6 @@ static int balance_nonroot( assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - j = 0; - for(i=0; i %d@%d", apNew[i]->pgno, - pNew->nCell+pNew->nOverflow, j, - cntNew[i] - iFirst, iFirst - ); - for(iCell=iFirst; iCellaData - || apCell[iCell]>=&pNew->aData[pBt->usableSize] - ){ - nCta += szCell[iCell]; - } - } - nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); - nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); - printf(" cta=%d free=%d\n", nCta, nFree); - if( i==(nNew-1) ){ - printf("-----\n"); - fflush(stdout); - } -#endif - - assert( inCell + apNew[i]->nOverflow; - aShiftRight[i] = cntNew[i] - j; - assert( i!=nOld-1 || j==nCell ); - if( j=0 && aShiftLeft[0]==0 ); for(i=0; i=nNew ? i-nNew : nNew-1-i); if( abDone[iPg]==0 - && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) - && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + && (iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1]) + && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) ){ int iNew; int iOld; From 40253262e0b3f9bfd79b6a57c891eb0f0459a38f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Oct 2014 21:35:05 +0000 Subject: [PATCH 06/51] Fix a (probably harmless) but in the CSV output mode of the command-line shell. FossilOrigin-Name: 19fe4a0a475bd94f491031aea7a183f7c0515cf3 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e210943d15..894c7cd51a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7 -D 2014-10-17T11:24:17.839 +C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. +D 2014-10-17T21:35:05.954 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 +F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1204,10 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 06c576c152c4013080c255cbbeb45bf2e298be9f -R 76bb09f8a8f8bdebeb708f5bb39aa31b -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7 * +P e4ab094f8afce0817f4074e823fabe59fc29ebb4 +R bacb142e035c538b5014a902fee951f4 U drh -Z b2f516d1147acb0e1bf1c700327ee52e +Z 99accb7d1ae416e10fed302750dd8f99 diff --git a/manifest.uuid b/manifest.uuid index f4f611fe44..db419ccbec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file +19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3ca4b094bc..59cd2011e7 100644 --- a/src/shell.c +++ b/src/shell.c @@ -882,7 +882,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int } fprintf(p->out,"%s",p->newline); } - if( azArg>0 ){ + if( nArg>0 ){ for(i=0; i Date: Tue, 21 Oct 2014 01:05:09 +0000 Subject: [PATCH 07/51] If a skip-scan is a proper subset of some other scan, then adjust the cost of the skip-scan upward so that it is more costly than the other scan. Such a cost imbalance can arise under STAT4 because of difficulties in getting an accurate estimate for skip-scans. FossilOrigin-Name: f4b22a2620a5dc48949048c2ecbd226755d4b2c3 --- manifest | 15 ++--- manifest.uuid | 2 +- src/where.c | 121 +++++++++++++++++------------------- src/whereInt.h | 4 +- test/skipscan6.test | 145 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 212 insertions(+), 75 deletions(-) create mode 100644 test/skipscan6.test diff --git a/manifest b/manifest index 894c7cd51a..3c0097dfe9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. -D 2014-10-17T21:35:05.954 +C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. +D 2014-10-21T01:05:09.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 -F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c +F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -844,6 +844,7 @@ F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 +F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d @@ -1204,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4ab094f8afce0817f4074e823fabe59fc29ebb4 -R bacb142e035c538b5014a902fee951f4 +P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 +R 7ae877aeccb485915279368cd2f54415 U drh -Z 99accb7d1ae416e10fed302750dd8f99 +Z dece11d7f7de853e09f0182395012c8f diff --git a/manifest.uuid b/manifest.uuid index db419ccbec..ee9b76f602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file +f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bc0110779e..00b1880880 100644 --- a/src/where.c +++ b/src/where.c @@ -2638,7 +2638,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; + nSkip = pLoop->nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2752,7 +2752,7 @@ static void explainAppendTerm( static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; + u16 nSkip = pLoop->nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; @@ -3189,7 +3189,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); + assert( nEq>=pLoop->nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -3206,7 +3206,7 @@ static Bitmask codeOneLoopStart( && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ - assert( pLoop->u.btree.nSkip==0 ); + assert( pLoop->nSkip==0 ); bSeekPastNull = 1; nExtraReg = 1; } @@ -3827,7 +3827,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } @@ -3956,12 +3956,15 @@ static int whereLoopCheaperProperSubset( const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -3983,25 +3986,12 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ @@ -4295,7 +4285,7 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ @@ -4324,7 +4314,7 @@ static int whereLoopAddBtreeIndex( pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; @@ -4332,44 +4322,6 @@ static int whereLoopAddBtreeIndex( pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } - pNew->nOut -= nIter; - /* TUNING: Because uncertainties in the estimates for skip-scan queries, - ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; - } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -4532,10 +4484,50 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + if( pTerm ){ + /* TUNING: When estimating skip-scan for a term that is also indexable, + ** multiply the cost of the skip-scan by 2.0, to make it a little less + ** desirable than the regular index lookup. */ + nIter += 10; assert( 10==sqlite3LogEst(2) ); + } + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + return rc; } @@ -4714,7 +4706,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -4755,7 +4747,7 @@ static int whereLoopAddBtree( } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -5305,7 +5297,7 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && pLoop->u.btree.nSkip==0 + && pLoop->nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5878,7 +5870,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; + pLoop->nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5890,7 +5882,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) diff --git a/src/whereInt.h b/src/whereInt.h index f17906e63a..e9eb8b7dd2 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -115,7 +115,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -128,12 +127,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a diff --git a/test/skipscan6.test b/test/skipscan6.test new file mode 100644 index 0000000000..9eda9a66f3 --- /dev/null +++ b/test/skipscan6.test @@ -0,0 +1,145 @@ +# 2014-10-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular, this file verifies that use of all columns of an index +# is always preferred over the use of a skip-scan on some columns of +# the same index. Because of difficulties in scoring a skip-scan, +# the skip-scan can sometimes come out with a lower raw score when +# using STAT4. But the query planner should detect this and use the +# full index rather than the skip-scan. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan6 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1( + aa int, + bb int, + cc int, + dd int, + ee int + ); + CREATE INDEX ix on t1(aa, bb, cc, dd DESC); + ANALYZE sqlite_master; + INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 1.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX ix .aa=. AND bb=../} + +do_execsql_test 2.1 { + DROP INDEX ix; + CREATE INDEX good on t1(bb, aa, dd DESC); + CREATE INDEX bad on t1(aa, bb, cc, dd DESC); + DELETE FROM sqlite_stat1; + DELETE FROM sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2'); + INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22'); + INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48'); + INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617'); + INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28'); + INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570'); + INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC'); + INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34'); + INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9'); + INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED'); + INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} + + + +finish_test From 1b131b7a7acc83c7bd1a5e7d0872c968d9e26489 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 16:01:40 +0000 Subject: [PATCH 08/51] Improvements to the WHERETRACE debugging logic. FossilOrigin-Name: ec1e942f08548695ff02645b3f3cd6bb2516bc9a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3c0097dfe9..d708dca4f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. -D 2014-10-21T01:05:09.795 +C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. +D 2014-10-21T16:01:40.774 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 -R 7ae877aeccb485915279368cd2f54415 +P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 +R 59b7c6284af5e31186eaa6edb85f9e59 U drh -Z dece11d7f7de853e09f0182395012c8f +Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b diff --git a/manifest.uuid b/manifest.uuid index ee9b76f602..838729f698 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file +ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 00b1880880..7dfe0f02bb 100644 --- a/src/where.c +++ b/src/where.c @@ -3995,11 +3995,15 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -5751,7 +5755,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; ii Date: Tue, 21 Oct 2014 18:16:21 +0000 Subject: [PATCH 09/51] Further tuning of the cost estimates for skip-scan loops, especially for cases when skip-scan loops are in competition with regular loops. FossilOrigin-Name: a27861c28c4791e51d797aa37e9cca806cb58775 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 10 +++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d708dca4f3..3daca49fdd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. -D 2014-10-21T16:01:40.774 +C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. +D 2014-10-21T18:16:21.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 +F src/where.c 994b38c8697aad095878ef1e4860902df457427f F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 -R 59b7c6284af5e31186eaa6edb85f9e59 +P ec1e942f08548695ff02645b3f3cd6bb2516bc9a +R 4f5d15d10f268e58d7c047d490bf8adc U drh -Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b +Z 7d7f3f0271fe8b8df2863a2c22053442 diff --git a/manifest.uuid b/manifest.uuid index 838729f698..52d4aafd99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file +a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7dfe0f02bb..7f51d00c87 100644 --- a/src/where.c +++ b/src/where.c @@ -3994,7 +3994,9 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. Except, do not adjust the cost estimate downward for + ** a loop that skips more columns. */ + if( pTemplate->nSkip>p->nSkip ) continue; WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; @@ -4515,12 +4517,6 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } pNew->nOut -= nIter; /* TUNING: Because uncertainties in the estimates for skip-scan queries, ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ From 442c5cd3cfc67d3e10aa64d9f180ef94fb3597cf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 21:56:06 +0000 Subject: [PATCH 10/51] Call fsync() right after ftruncate() when in journal_mode=TRUNCATE and when synchronous=FULL in order to ensure that transactions are durable across a power loss that happens moments after the commit. Proposed fix for [https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. FossilOrigin-Name: 3e922208b68563489c7766abb9afb4885113e7b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3daca49fdd..f4a8a2a556 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. -D 2014-10-21T18:16:21.388 +C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. +D 2014-10-21T21:56:06.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 +F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec1e942f08548695ff02645b3f3cd6bb2516bc9a -R 4f5d15d10f268e58d7c047d490bf8adc +P a27861c28c4791e51d797aa37e9cca806cb58775 +R 1375151e695773da263f40095fae22af U drh -Z 7d7f3f0271fe8b8df2863a2c22053442 +Z be5219411fed69e2fe0e0626925bcd5e diff --git a/manifest.uuid b/manifest.uuid index 52d4aafd99..db15d6664e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file +3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d3a36ef484..d840a39a15 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST From 9f07cf7b2e43548f60bd7497dafba30856df3531 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 15:27:05 +0000 Subject: [PATCH 11/51] Take steps to avoid misestimating range query costs based on STAT4 data due to the roundoff error of converting from integers to LogEst and back to integers. FossilOrigin-Name: 3c933bf95f291f7957580d823dce92c981375a5c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 1 + src/sqliteInt.h | 3 ++- src/where.c | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f4a8a2a556..8290b14522 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. -D 2014-10-21T21:56:06.890 +C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. +D 2014-10-22T15:27:05.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d +F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 +F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 994b38c8697aad095878ef1e4860902df457427f +F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a27861c28c4791e51d797aa37e9cca806cb58775 -R 1375151e695773da263f40095fae22af +P 3e922208b68563489c7766abb9afb4885113e7b8 +R 5c508733aebac553de79291a7eabd709 U drh -Z be5219411fed69e2fe0e0626925bcd5e +Z a3d7104c7f173585d7e788d6453aa065 diff --git a/manifest.uuid b/manifest.uuid index db15d6664e..b84af28cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file +3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7d36f01318..67bba1a28a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1599,6 +1599,7 @@ static void initAvgEq(Index *pIdx){ nRow = pIdx->aiRowEst[0]; nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that ** occur in the stat4 table for this index. Set sumEq to the sum of diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cba89b03e7..5409f7086f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1801,7 +1801,8 @@ struct Index { int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ - tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; diff --git a/src/where.c b/src/where.c index 7f51d00c87..f708d7fad7 100644 --- a/src/where.c +++ b/src/where.c @@ -2198,7 +2198,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + iUpper = p->nRowEst0; }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ From d7d71470226bcc59799713b95ad9e10f9eff63cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 19:57:16 +0000 Subject: [PATCH 12/51] Change the 0x800 bit of SQLITE_TESTCTRL_OPTIMIZATIONS so that it disables the loading of STAT3 and STAT4 content, not just the using of that content. Change the internal name of that bit to SQLITE_Stat34. FossilOrigin-Name: ca3b00c44ec52d209642a5ba9ef82e085fac39db --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 2 +- src/sqliteInt.h | 2 +- src/test1.c | 3 ++- src/where.c | 4 +--- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 8290b14522..0f41135502 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. -D 2014-10-22T15:27:05.734 +C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. +D 2014-10-22T19:57:16.520 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 +F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,12 +232,12 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc +F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 518db4305d76b29dd9da3f022ca899c8fcdf9fc7 +F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 +F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e922208b68563489c7766abb9afb4885113e7b8 -R 5c508733aebac553de79291a7eabd709 +P 3c933bf95f291f7957580d823dce92c981375a5c +R 3efbbff265eea57d45c364e4606463ea U drh -Z a3d7104c7f173585d7e788d6453aa065 +Z b4247f0ab9b19770a6354621b6169ee4 diff --git a/manifest.uuid b/manifest.uuid index b84af28cee..fff2e6a3fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file +ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 67bba1a28a..597885237c 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1861,7 +1861,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5409f7086f..1b3138be44 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1214,7 +1214,7 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/test1.c b/src/test1.c index 85a16488ba..802aa99b5b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6293,7 +6293,8 @@ static int optimization_control( { "transitive", SQLITE_Transitive }, { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, - { "stat3", SQLITE_Stat3 }, + { "stat3", SQLITE_Stat34 }, + { "stat4", SQLITE_Stat34 }, }; if( objc!=4 ){ diff --git a/src/where.c b/src/where.c index f708d7fad7..198a6c3fdd 100644 --- a/src/where.c +++ b/src/where.c @@ -1350,7 +1350,7 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -2159,7 +2159,6 @@ static int whereRangeScanEst( if( p->nSample>0 && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; @@ -4422,7 +4421,6 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; From b7288e28e48235e8f7671ae7b613f4c5c7251c44 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 20:07:19 +0000 Subject: [PATCH 13/51] Disable the use of strchrnul() unless specifically enabled by compile-time options. FossilOrigin-Name: e580470db77d6da970c755102790e603fb26b3c6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0f41135502..ae5d008c78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. -D 2014-10-22T19:57:16.520 +C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. +D 2014-10-22T20:07:19.558 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 +F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c933bf95f291f7957580d823dce92c981375a5c -R 3efbbff265eea57d45c364e4606463ea +P ca3b00c44ec52d209642a5ba9ef82e085fac39db +R 8aff7924cf3a3f93f4ee99fb545d1f5f U drh -Z b4247f0ab9b19770a6354621b6169ee4 +Z c6210337ac863b7dfe5cbe19c541a0aa diff --git a/manifest.uuid b/manifest.uuid index fff2e6a3fa..857c7b0e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file +e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index c0b3c70f6b..1df287fbb6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,11 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(linux) -# define HAVE_STRCHRNUL 1 -# else -# define HAVE_STRCHRNUL 0 -# endif +# define HAVE_STRCHRNUL 0 #endif From 4f81bbb5289cdd248c21775b9e4cdb92e110e139 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Oct 2014 01:01:26 +0000 Subject: [PATCH 14/51] Implement sqlite3_create_collation() by invoking sqlite3_create_collation_v2() with a NULL destructor argument. This saves a little space. FossilOrigin-Name: 9762ad0639cca2fc1ef0573113fb613ce9e7e83e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 8 +------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ae5d008c78..5b87a53662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. -D 2014-10-22T20:07:19.558 +C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. +D 2014-10-23T01:01:26.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 +F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca3b00c44ec52d209642a5ba9ef82e085fac39db -R 8aff7924cf3a3f93f4ee99fb545d1f5f +P e580470db77d6da970c755102790e603fb26b3c6 +R f159389d59e4350424d5f438676d787b U drh -Z c6210337ac863b7dfe5cbe19c541a0aa +Z 177632e852e703fce9e831bfbe27f418 diff --git a/manifest.uuid b/manifest.uuid index 857c7b0e10..b0a7f1f074 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file +9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file diff --git a/src/main.c b/src/main.c index ea03f2639f..e6cb4cd3df 100644 --- a/src/main.c +++ b/src/main.c @@ -2751,13 +2751,7 @@ int sqlite3_create_collation( void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); } /* From 9ca95730e3a25bf5b335ec73a0a14f0112a421fb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 00:35:58 +0000 Subject: [PATCH 15/51] Add the SQLITE_ENABLE_API_ARMOR compile-time option. This is a work in progress and is not yet completely functional. FossilOrigin-Name: c297a84bc678f81ffc0aa9139ab73f0ca87c1971 --- manifest | 49 +++++++------- manifest.uuid | 2 +- src/auth.c | 3 + src/backup.c | 22 +++++++ src/build.c | 6 +- src/complete.c | 7 ++ src/ctime.c | 10 +++ src/main.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- src/malloc.c | 3 + src/mutex_unix.c | 8 ++- src/os.c | 4 ++ src/prepare.c | 13 ++-- src/printf.c | 21 ++++++ src/random.c | 11 ++-- src/status.c | 8 +++ src/table.c | 3 + src/util.c | 10 +++ src/vdbeapi.c | 31 +++++++-- src/vdbeblob.c | 5 ++ src/vtab.c | 16 ++++- 20 files changed, 351 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..0b02968a4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption.\s\sThis\sis\sa\swork\sin\nprogress\sand\sis\snot\syet\scompletely\sfunctional. +D 2014-10-24T00:35:58.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,17 +168,17 @@ F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 -F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 -F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e +F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 +F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 -F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 +F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 -F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 -F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 +F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 +F src/ctime.c dfa83bfebb4201d07b16534acb8a0149592c3a25 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d @@ -194,8 +194,8 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 -F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec +F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 +F src/malloc.c 5e02eab3e70cfcc265817f4de008f55e768dca51 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -205,10 +205,10 @@ F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 -F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 +F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace +F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa @@ -222,9 +222,9 @@ F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f -F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 -F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece +F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 +F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 +F src/random.c 689c95a50b90e7a8b3e6088e33d1d2d4212e6e88 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -234,8 +234,8 @@ F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb -F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb +F src/status.c 81712116e826b0089bb221b018929536b2b5406f +F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 @@ -287,18 +287,18 @@ F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c -F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 +F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 -F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 +F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 -F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 +F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de +F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 @@ -1205,7 +1205,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R 98ebc598759882f163c17892cc0d9e3e +T *branch * api-armor +T *sym-api-armor * +T -sym-trunk * U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 48418cab28475295c0204ad2ebe00aca diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..64b7efe90b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +c297a84bc678f81ffc0aa9139ab73f0ca87c1971 \ No newline at end of file diff --git a/src/auth.c b/src/auth.c index 1680c9a7c2..9768fc2fc0 100644 --- a/src/auth.c +++ b/src/auth.c @@ -72,6 +72,9 @@ int sqlite3_set_authorizer( int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; diff --git a/src/backup.c b/src/backup.c index 92c6334bde..da4303e5fd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -138,6 +138,13 @@ sqlite3_backup *sqlite3_backup_init( ){ sqlite3_backup *p; /* Value to return */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + /* Lock the source database handle. The destination database ** handle is not locked in this routine, but it is locked in ** sqlite3_backup_step(). The user is required to ensure that no @@ -334,6 +341,9 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ int pgszSrc = 0; /* Source page size */ int pgszDest = 0; /* Destination page size */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); if( p->pDestDb ){ @@ -623,6 +633,12 @@ int sqlite3_backup_finish(sqlite3_backup *p){ ** call to sqlite3_backup_step(). */ int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nRemaining; } @@ -631,6 +647,12 @@ int sqlite3_backup_remaining(sqlite3_backup *p){ ** recent call to sqlite3_backup_step(). */ int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nPagecount; } diff --git a/src/build.c b/src/build.c index b897494db3..0b4affc664 100644 --- a/src/build.c +++ b/src/build.c @@ -307,7 +307,11 @@ int sqlite3UserAuthTable(const char *zTable){ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - assert( zName!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return 0; +#endif + /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); #if SQLITE_USER_AUTHENTICATION diff --git a/src/complete.c b/src/complete.c index 6ab6f4a042..c439cfe181 100644 --- a/src/complete.c +++ b/src/complete.c @@ -105,6 +105,13 @@ int sqlite3_complete(const char *zSql){ u8 state = 0; /* Current state, using numbers defined in header comment */ u8 token; /* Value of the next token */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + #ifndef SQLITE_OMIT_TRIGGER /* A complex statement machine used to detect the end of a CREATE TRIGGER ** statement. This is the normal case. diff --git a/src/ctime.c b/src/ctime.c index 82a2f35204..36d0c266ee 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -63,6 +63,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#ifdef SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif @@ -388,6 +391,13 @@ static const char * const azCompileOpt[] = { */ int sqlite3_compileoption_used(const char *zOptName){ int i, n; + +#ifdef SQLITE_ENABLE_API_ARMORE + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); diff --git a/src/main.c b/src/main.c index e6cb4cd3df..5308698797 100644 --- a/src/main.c +++ b/src/main.c @@ -598,6 +598,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ ** Return the mutex associated with a database connection. */ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->mutex; } @@ -607,6 +613,10 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ */ int sqlite3_db_release_memory(sqlite3 *db){ int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -737,6 +747,12 @@ static int nocaseCollatingFunc( ** Return the ROWID of the most recent insert */ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->lastRowid; } @@ -744,6 +760,12 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ ** Return the number of changes in the most recent call to sqlite3_exec(). */ int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nChange; } @@ -751,6 +773,12 @@ int sqlite3_changes(sqlite3 *db){ ** Return the number of changes since the database handle was opened. */ int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nTotalChange; } @@ -1296,6 +1324,9 @@ int sqlite3_busy_handler( int (*xBusy)(void*,int), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE; +#endif sqlite3_mutex_enter(db->mutex); db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; @@ -1317,6 +1348,12 @@ void sqlite3_progress_handler( int (*xProgress)(void*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif sqlite3_mutex_enter(db->mutex); if( nOps>0 ){ db->xProgress = xProgress; @@ -1337,6 +1374,9 @@ void sqlite3_progress_handler( ** specified number of milliseconds before returning 0. */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( ms>0 ){ sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; @@ -1350,6 +1390,12 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ ** Cause any pending operation to stop at its earliest opportunity. */ void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif db->u1.isInterrupted = 1; } @@ -1487,6 +1533,12 @@ int sqlite3_create_function_v2( ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( xDestroy ){ pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); @@ -1523,6 +1575,10 @@ int sqlite3_create_function16( ){ int rc; char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); @@ -1554,6 +1610,12 @@ int sqlite3_overload_function( ){ int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, @@ -1575,6 +1637,13 @@ int sqlite3_overload_function( */ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->xTrace = xTrace; @@ -1596,6 +1665,13 @@ void *sqlite3_profile( void *pArg ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pProfileArg; db->xProfile = xProfile; @@ -1616,6 +1692,13 @@ void *sqlite3_commit_hook( void *pArg /* Argument to the function */ ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pCommitArg; db->xCommitCallback = xCallback; @@ -1634,6 +1717,13 @@ void *sqlite3_update_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pUpdateArg; db->xUpdateCallback = xCallback; @@ -1652,6 +1742,13 @@ void *sqlite3_rollback_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pRollbackArg; db->xRollbackCallback = xCallback; @@ -1698,6 +1795,9 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ UNUSED_PARAMETER(db); UNUSED_PARAMETER(nFrame); #else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -1718,6 +1818,12 @@ void *sqlite3_wal_hook( ){ #ifndef SQLITE_OMIT_WAL void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; db->xWalCallback = xCallback; @@ -1745,6 +1851,10 @@ int sqlite3_wal_checkpoint_v2( int rc; /* Return code */ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; @@ -2141,6 +2251,12 @@ static const int aHardLimit[] = { int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME ** there is a hard upper bound set at compile-time by a C preprocessor @@ -2426,6 +2542,9 @@ static int openDatabase( char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -2715,13 +2834,15 @@ int sqlite3_open16( sqlite3_value *pVal; int rc; - assert( zFilename ); - assert( ppDb ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; #endif + if( zFilename==0 ) zFilename = "\000\000"; pVal = sqlite3ValueNew(0); sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); @@ -2766,6 +2887,10 @@ int sqlite3_create_collation_v2( void(*xDel)(void*) ){ int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); @@ -2787,6 +2912,10 @@ int sqlite3_create_collation16( ){ int rc = SQLITE_OK; char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); @@ -2809,6 +2938,9 @@ int sqlite3_collation_needed( void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = xCollNeeded; db->xCollNeeded16 = 0; @@ -2827,6 +2959,9 @@ int sqlite3_collation_needed16( void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = 0; db->xCollNeeded16 = xCollNeeded16; @@ -2853,6 +2988,12 @@ int sqlite3_global_recover(void){ ** by the next COMMIT or ROLLBACK. */ int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->autoCommit; } @@ -3035,6 +3176,9 @@ int sqlite3_sleep(int ms){ ** Enable or disable the extended result codes. */ int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->errMask = onoff ? 0xffffffff : 0xff; sqlite3_mutex_leave(db->mutex); @@ -3048,6 +3192,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; Btree *pBtree; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); pBtree = sqlite3DbNameToBtree(db, zDbName); if( pBtree ){ @@ -3390,7 +3537,7 @@ int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 ) return 0; + if( zFilename==0 || zParam==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -3446,6 +3593,12 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ ** connection. */ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } @@ -3455,6 +3608,12 @@ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ ** no such database exists. */ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } diff --git a/src/malloc.c b/src/malloc.c index 6fb9d53d1b..c3c644fd6a 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -475,6 +475,9 @@ sqlite3_uint64 sqlite3_msize(void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ +#if defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_OMIT_AUTOINIT) + if( sqlite3_initialize() ) return; +#endif if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); diff --git a/src/mutex_unix.c b/src/mutex_unix.c index c8663144e8..c936914d8a 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -175,8 +175,12 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(staticMutexes) ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif p = &staticMutexes[iType-2]; #if SQLITE_MUTEX_NREF p->id = iType; diff --git a/src/os.c b/src/os.c index b6c28a1dc4..2a2cf13c5e 100644 --- a/src/os.c +++ b/src/os.c @@ -361,6 +361,10 @@ int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ int rc = sqlite3_initialize(); if( rc ) return rc; #endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); diff --git a/src/prepare.c b/src/prepare.c index a05e619f3e..ca9c64b441 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -709,9 +709,12 @@ static int sqlite3LockAndPrepare( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - assert( ppStmt!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -818,9 +821,11 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - assert( ppStmt ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } if( nBytes>=0 ){ diff --git a/src/printf.c b/src/printf.c index 1df287fbb6..f000da7fcc 100644 --- a/src/printf.c +++ b/src/printf.c @@ -223,6 +223,13 @@ void sqlite3VXPrintf( PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ap==0 ){ + (void)SQLITE_MISUSE_BKPT; + sqlite3StrAccumReset(pAccum); + return; + } +#endif bufpt = 0; if( bFlags ){ if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ @@ -943,6 +950,13 @@ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -985,6 +999,13 @@ char *sqlite3_mprintf(const char *zFormat, ...){ char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf && n>0 ) zBuf[0] = 0; + return zBuf; + } +#endif sqlite3StrAccumInit(&acc, zBuf, n, 0); acc.useMalloc = 0; sqlite3VXPrintf(&acc, 0, zFormat, ap); diff --git a/src/random.c b/src/random.c index b82566524c..7bca0d71af 100644 --- a/src/random.c +++ b/src/random.c @@ -34,6 +34,11 @@ void sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + if( pBuf==0 || N<=0 ) return; + /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, ** we have to locate the state vector at run-time. In the more common @@ -52,12 +57,6 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif - if( N<=0 ){ - wsdPrng.isInit = 0; - sqlite3_mutex_leave(mutex); - return; - } - /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not diff --git a/src/status.c b/src/status.c index 79a8001b8a..4c2eabb661 100644 --- a/src/status.c +++ b/src/status.c @@ -86,6 +86,9 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ return SQLITE_MISUSE_BKPT; } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; if( resetFlag ){ @@ -105,6 +108,11 @@ int sqlite3_db_status( int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { diff --git a/src/table.c b/src/table.c index c435b2bc02..6e1df30643 100644 --- a/src/table.c +++ b/src/table.c @@ -126,6 +126,9 @@ int sqlite3_get_table( int rc; TabResult res; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; if( pnRow ) *pnRow = 0; diff --git a/src/util.c b/src/util.c index 9bb8d89157..ab409fa256 100644 --- a/src/util.c +++ b/src/util.c @@ -251,6 +251,11 @@ int sqlite3Dequote(char *z){ */ int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -258,6 +263,11 @@ int sqlite3_stricmp(const char *zLeft, const char *zRight){ } int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 0ab76e0784..daf7b101d2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -966,11 +966,19 @@ static const void *columnName( const void *(*xFunc)(Mem*), int useType ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; + const void *ret; + Vdbe *p; int n; - sqlite3 *db = p->db; - + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ @@ -1435,6 +1443,12 @@ int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ */ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(pDb->mutex); if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; @@ -1450,7 +1464,14 @@ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ */ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v = pVdbe->aCounter[op]; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + v = pVdbe->aCounter[op]; if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 71bd8816d5..0cf2b0652b 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -153,6 +153,11 @@ int sqlite3_blob_open( Parse *pParse = 0; Incrblob *pBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; diff --git a/src/vtab.c b/src/vtab.c index faee4ae478..334de9aacb 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -81,6 +81,9 @@ int sqlite3_create_module( const sqlite3_module *pModule, /* The definition of the module */ void *pAux /* Context pointer for xCreate/xConnect */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, 0); } @@ -94,6 +97,9 @@ int sqlite3_create_module_v2( void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, xDestroy); } @@ -698,6 +704,9 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; char *zErr = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ sqlite3Error(db, SQLITE_MISUSE); @@ -1054,6 +1063,9 @@ int sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); assert( OE_Ignore==4 && OE_Replace==5 ); assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); @@ -1069,8 +1081,10 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); switch( op ){ case SQLITE_VTAB_CONSTRAINT_SUPPORT: { From 5a5d120bcc08af809b8687ff9fe39fce27e7796c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 12:37:00 +0000 Subject: [PATCH 16/51] Fix two problems. Tests now passing. FossilOrigin-Name: 1c220b806d56e163842e17038c3331f71861bd9c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/malloc.c | 3 --- src/random.c | 7 ++++++- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 0b02968a4f..da62517a33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption.\s\sThis\sis\sa\swork\sin\nprogress\sand\sis\snot\syet\scompletely\sfunctional. -D 2014-10-24T00:35:58.052 +C Fix\stwo\sproblems.\s\sTests\snow\spassing. +D 2014-10-24T12:37:00.827 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 -F src/malloc.c 5e02eab3e70cfcc265817f4de008f55e768dca51 +F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c 689c95a50b90e7a8b3e6088e33d1d2d4212e6e88 +F src/random.c b8a058131851de1a37801b5587845ee73411c064 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1205,10 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R 98ebc598759882f163c17892cc0d9e3e -T *branch * api-armor -T *sym-api-armor * -T -sym-trunk * +P c297a84bc678f81ffc0aa9139ab73f0ca87c1971 +R 1cc86c78be65c50602286adf14ad3d43 U drh -Z 48418cab28475295c0204ad2ebe00aca +Z 4a2cbf72bfaa58a272d60444a54fd92d diff --git a/manifest.uuid b/manifest.uuid index 64b7efe90b..0375d856fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c297a84bc678f81ffc0aa9139ab73f0ca87c1971 \ No newline at end of file +1c220b806d56e163842e17038c3331f71861bd9c \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index c3c644fd6a..6fb9d53d1b 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -475,9 +475,6 @@ sqlite3_uint64 sqlite3_msize(void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ -#if defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_OMIT_AUTOINIT) - if( sqlite3_initialize() ) return; -#endif if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); diff --git a/src/random.c b/src/random.c index 7bca0d71af..c4241362de 100644 --- a/src/random.c +++ b/src/random.c @@ -37,7 +37,6 @@ void sqlite3_randomness(int N, void *pBuf){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return; #endif - if( pBuf==0 || N<=0 ) return; /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, @@ -57,6 +56,12 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif + if( N<=0 || pBuf==0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not From 9769efcc4c785dc7552a2e22782f12e3c56e0169 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 14:32:21 +0000 Subject: [PATCH 17/51] Get the likelihood() functions working on operators like BETWEEN that create virtual terms in the WHERE-clause analysis. FossilOrigin-Name: 03d0498d0f24bec2383d5d79edf25069effecd59 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 ++++++++++++++++------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..67132856cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. +D 2014-10-24T14:32:21.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 +F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R f460b87a81ee1fb775b7700e4252ece6 U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 1428b6fe81396b49c1c6b7022fde286d diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..59bf5ae4e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 198a6c3fdd..d4969fad1e 100644 --- a/src/where.c +++ b/src/where.c @@ -756,6 +756,15 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ } } +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* ** Analyze a term that consists of two or more OR-connected @@ -1053,8 +1062,7 @@ static void exprAnalyzeOrTerm( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; + markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } @@ -1156,9 +1164,8 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew==0 ) return; pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; if( pExpr->op==TK_EQ && !ExprHasProperty(pExpr, EP_FromJoin) @@ -1215,9 +1222,8 @@ static void exprAnalyze( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); } - pTerm->nChild = 2; } #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ @@ -1292,9 +1298,8 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -1327,9 +1332,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -1369,9 +1373,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } From 4dd96a8315b3f3b2a516513b06754e1887e426eb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 15:26:29 +0000 Subject: [PATCH 18/51] Honor a high likelihood() on range constraints. FossilOrigin-Name: 401235edf40fcd665eaf426cf5155ac6855e8537 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 67132856cf..e98be71834 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. -D 2014-10-24T14:32:21.050 +C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. +D 2014-10-24T15:26:29.800 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 +F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R f460b87a81ee1fb775b7700e4252ece6 +P 03d0498d0f24bec2383d5d79edf25069effecd59 +R eb013b99b9ff12a41d5ffc947e4a6227 U drh -Z 1428b6fe81396b49c1c6b7022fde286d +Z 3215dd92b9608b7b9136fb91eb8178e8 diff --git a/manifest.uuid b/manifest.uuid index 59bf5ae4e6..4ee03d34ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file +401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d4969fad1e..e13b223366 100644 --- a/src/where.c +++ b/src/where.c @@ -2275,12 +2275,15 @@ static int whereRangeScanEst( nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); - /* TUNING: If there is both an upper and lower limit, assume the range is + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; From e6593d8e8c51fd05c762f34c444b19a74137fedc Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 16:40:49 +0000 Subject: [PATCH 19/51] Fix some minor formatting and code organization issues. FossilOrigin-Name: eab8706dc47aa0a44caf73619de858397c3e0b4e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 16 +++++++--------- src/pager.c | 15 +++++++++++---- src/pager.h | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 396fe366d0..b52c190c13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\swith\sthis\sbranch. -D 2014-10-22T18:42:31.680 +C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. +D 2014-10-24T16:40:49.448 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 37229f3134416f24828af823a3a6f8535923d45b +F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 18be49e363106bdb4c982fa27dbee7a8cd297113 -F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 +F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba 3c933bf95f291f7957580d823dce92c981375a5c -R 738ce949461e4c95ef4d450eb06d976a +P 854a54c6c21e800b0cd999023014813f7c50b23f +R b3aec92126a25950ff876c6cdf2fb4df U dan -Z 1bc59f5d05ced43bcf6fa4690adf0fcc +Z 9e7a15bc5953228881d999c99d9f0682 diff --git a/manifest.uuid b/manifest.uuid index b48e7f9fc1..5987e57b67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -854a54c6c21e800b0cd999023014813f7c50b23f \ No newline at end of file +eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d1457c5f39..b5e8cb2e0c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1233,7 +1233,7 @@ static int defragmentPage(MemPage *pPage){ ** ** This function may detect corruption within pPg. If it does and argument ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ @@ -6345,7 +6345,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 1 +#if 0 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6522,10 +6522,9 @@ static int balance_nonroot( u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - - u8 abDone[NB+2]; - Pgno aPgno[NB+2]; - u16 aPgFlags[NB+2]; + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; @@ -6868,8 +6867,7 @@ static int balance_nonroot( } } if( apNew[i]->pgno!=iMin ){ - apNew[i]->pDbPage->flags = flags; - sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } } @@ -7090,7 +7088,7 @@ static int balance_nonroot( freePage(apOld[i], &rc); } -#if 1 +#if 0 if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while diff --git a/src/pager.c b/src/pager.c index 74667afab0..dc79e6754a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6845,16 +6845,23 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } +#endif -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ PgHdr *pPg = (PgHdr*)pPage; - assert( pPg->flags & PGHDR_DIRTY ); + assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); assert( !subjRequiresPage(pPg) ); + assert( pPg->pgno!=iNew ); + pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); } -#endif - /* ** Return a pointer to the data for the specified page. */ diff --git a/src/pager.h b/src/pager.h index f3a04bbca6..764c7bd197 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,7 +188,7 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -void sqlite3PagerRekey(DbPage*, Pgno); +void sqlite3PagerRekey(DbPage*, Pgno, u16); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 23eba45d23291617e5f69ffb15c0cfbb1bab5121 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 18:43:57 +0000 Subject: [PATCH 20/51] Fix some issues in the new code on this branch related to the handling of corrupt databases. FossilOrigin-Name: 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 14 ++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b52c190c13..10be86a2cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. -D 2014-10-24T16:40:49.448 +C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. +D 2014-10-24T18:43:57.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 +F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 854a54c6c21e800b0cd999023014813f7c50b23f -R b3aec92126a25950ff876c6cdf2fb4df +P eab8706dc47aa0a44caf73619de858397c3e0b4e +R 8a34c36d0ed8bbbc0bb7360d0dc30d47 U dan -Z 9e7a15bc5953228881d999c99d9f0682 +Z 610a2814ec6cdf214c41e43dd2355678 diff --git a/manifest.uuid b/manifest.uuid index 5987e57b67..11462e357c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file +19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b5e8cb2e0c..f326f713fb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1386,7 +1386,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ assert( iStart<=iLast ); @@ -6065,7 +6065,7 @@ static int pageInsertArray( u8 *aData = pPg->aData; u8 *pData = *ppData; const int bFreelist = aData[1] || aData[2]; - assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; iaCellIdx[i*2]); if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ @@ -6866,7 +6866,13 @@ static int balance_nonroot( flags = aPgFlags[j]; } } - if( apNew[i]->pgno!=iMin ){ + if( iMin==0 ){ + /* This case can only occur if aPgno[] contains duplicate page + ** numbers. Which can only happen if the database is corrupt. */ + assert( CORRUPT_DB ); + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + }else if( apNew[i]->pgno!=iMin ){ sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } From 059b2d50e1c6a57ca301f3c9639f92f7e16ff96e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 19:28:09 +0000 Subject: [PATCH 21/51] Enhance the automatic index logic so that it creates a partial index when doing so gives the same answer for less work. FossilOrigin-Name: d95d0313c447f5baeabdb17284d8606331ab7d49 --- manifest | 19 +++++----- manifest.uuid | 2 +- src/expr.c | 85 +++++++++++++++++++++++++++----------------- src/resolve.c | 4 +-- src/sqliteInt.h | 5 ++- src/where.c | 23 ++++++++++-- test/autoindex4.test | 52 +++++++++++++++++++++++++++ 7 files changed, 143 insertions(+), 47 deletions(-) create mode 100644 test/autoindex4.test diff --git a/manifest b/manifest index e98be71834..e831ecb526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. -D 2014-10-24T15:26:29.800 +C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. +D 2014-10-24T19:28:09.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d +F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 +F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d +F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 +F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -346,6 +346,7 @@ F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1205,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 03d0498d0f24bec2383d5d79edf25069effecd59 -R eb013b99b9ff12a41d5ffc947e4a6227 +P 401235edf40fcd665eaf426cf5155ac6855e8537 +R def8f7d87d0f5784d9eaab2c56b2a690 U drh -Z 3215dd92b9608b7b9136fb91eb8178e8 +Z cb611c4b71e8fc233ea4e1005f56a9e0 diff --git a/manifest.uuid b/manifest.uuid index 4ee03d34ab..7417ef4fc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file +d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1ad9a879a3..13a9cb46fd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1210,20 +1210,24 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.i to 0 if the expression is -** not constant. +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() pWalker->u.i==1 -** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 -** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. ** ** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing -** an existing schema and 3 when processing a new statement. A bound +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound ** parameter raises an error for new statements, but is silently converted ** to NULL for existing schemas. This allows sqlite_master tables that ** contain a bound parameter because they were generated by older versions @@ -1232,23 +1236,25 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 2 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -1257,18 +1263,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } case TK_VARIABLE: - if( pWalker->u.i==4 ){ + if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE ** statements into a NULL when parsing the CREATE statement text out ** of the sqlite_master table */ pExpr->op = TK_NULL; - }else if( pWalker->u.i==3 ){ + }else if( pWalker->eCode==4 ){ /* A bound parameter in a CREATE statement that originates from ** sqlite3_prepare() causes an error */ - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } /* Fall through */ @@ -1280,21 +1290,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -1302,21 +1313,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2); + return exprIsConst(p, 2, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -1326,7 +1347,7 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 3+isInit); + return exprIsConst(p, 4+isInit, 0); } /* diff --git a/src/resolve.c b/src/resolve.c index d6a865caef..e507ccb810 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -28,7 +28,7 @@ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -36,7 +36,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1b3138be44..b7f992c34d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2892,9 +2892,11 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ } u; @@ -3295,6 +3297,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); +int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); diff --git a/src/where.c b/src/where.c index e13b223366..8d2a891870 100644 --- a/src/where.c +++ b/src/where.c @@ -1594,6 +1594,8 @@ static void constructAutomaticIndex( Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -1609,6 +1611,11 @@ static void constructAutomaticIndex( pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -1621,7 +1628,9 @@ static void constructAutomaticIndex( sentWarning = 1; } if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } @@ -1654,7 +1663,7 @@ static void constructAutomaticIndex( /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; + if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; pIdx->pTable = pTable; @@ -1706,18 +1715,28 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ diff --git a/test/autoindex4.test b/test/autoindex4.test new file mode 100644 index 0000000000..6d0865bf72 --- /dev/null +++ b/test/autoindex4.test @@ -0,0 +1,52 @@ +# 2014-10-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file implements regression tests for SQLite library. The +# focus of this script is testing automatic index creation logic, +# and specifically creation of automatic partial indexes. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test autoindex4-1.0 { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(123,'abc'),(234,'def'),(234,'ghi'),(345,'jkl'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(987,'zyx'),(654,'wvu'),(987,'rqp'); + + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=987 ORDER BY +b; +} {234 def 987 rqp | 234 def 987 zyx | 234 ghi 987 rqp | 234 ghi 987 zyx |} +do_execsql_test autoindex4-1.1 { + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=555; +} {} + +do_execsql_test autoindex4-1.2 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON a=234 AND x=555; +} {123 abc {} {} | 234 def {} {} | 234 ghi {} {} | 345 jkl {} {} |} +do_execsql_test autoindex4-1.3 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON x=555 WHERE a=234; +} {234 def {} {} | 234 ghi {} {} |} +do_execsql_test autoindex4-1.4 { + SELECT *, '|' FROM t1 LEFT JOIN t2 WHERE a=234 AND x=555; +} {} + + +do_execsql_test autoindex4-2.0 { + CREATE TABLE t3(e,f); + INSERT INTO t3 VALUES(123,654),(555,444),(234,987); + + SELECT (SELECT count(*) FROM t1, t2 WHERE a=e AND x=f), e, f, '|' + FROM t3 + ORDER BY rowid; +} {1 123 654 | 0 555 444 | 4 234 987 |} + +finish_test From 31f4e99d449b5938f46723f6d2ec96595ff88513 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 20:57:03 +0000 Subject: [PATCH 22/51] Ensure that the "Any prior cache entry associated with newKey is guaranteed not to be pinned" guarantee made to xRekey implementations is not violated. FossilOrigin-Name: ecc3544e712041736af7c7b4f34864a1f2e30ff7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 30 ++++++++++++------------------ src/pager.c | 5 +---- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 10be86a2cc..6c0464217b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. -D 2014-10-24T18:43:57.313 +C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. +D 2014-10-24T20:57:03.500 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea +F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eab8706dc47aa0a44caf73619de858397c3e0b4e -R 8a34c36d0ed8bbbc0bb7360d0dc30d47 +P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc +R c9d1872515b77329bc9fbfe4c8e91668 U dan -Z 610a2814ec6cdf214c41e43dd2355678 +Z 931e363227b1144b3ac19bee87a82036 diff --git a/manifest.uuid b/manifest.uuid index 11462e357c..470b2345ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file +ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f326f713fb..83ed66414c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6856,25 +6856,19 @@ static int balance_nonroot( aPgFlags[i] = apNew[i]->pDbPage->flags; } for(i=0; ipgno); - Pgno iMin = 0; - u16 flags = 0; - for(j=0; jiGt && (iMin==0 || iPgnopgno!=iMin ){ - sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); - apNew[i]->pgno = iMin; + pgno = aPgno[iBest]; + aPgno[iBest] = 0xffffffff; + if( iBest!=i ){ + if( iBest>i ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } diff --git a/src/pager.c b/src/pager.c index dc79e6754a..997f842d0a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6853,10 +6853,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** page number to iNew and sets the value of the PgHdr.flags field to ** the value passed as the third parameter. */ -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ - PgHdr *pPg = (PgHdr*)pPage; - assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); - assert( !subjRequiresPage(pPg) ); +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ assert( pPg->pgno!=iNew ); pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); From 051575cbf4c47fc472f53146e30d6ddddc9054b3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 12:28:25 +0000 Subject: [PATCH 23/51] Do not use virtual (and hence redundant) WHERE-clause terms to restrict the content of a automatic partial index. Show when an automatic partial index is used in the EXPLAIN QUERY PLAN output. FossilOrigin-Name: b9ad601eab1d7298d369267eb697c7fa1bc16985 --- ext/rtree/rtree6.test | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 4 ++++ src/whereInt.h | 1 + 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index cec3a8da41..c9c87e8ad9 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -102,7 +102,7 @@ do_eqp_test rtree6.2.4.2 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} - 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { diff --git a/manifest b/manifest index e831ecb526..28fa8e1387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. -D 2014-10-24T19:28:09.216 +C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. +D 2014-10-25T12:28:25.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test 0cfbdf27ee086bf16a3da2c6f2d5b3d6473cb27e +F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 -F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 +F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 401235edf40fcd665eaf426cf5155ac6855e8537 -R def8f7d87d0f5784d9eaab2c56b2a690 +P d95d0313c447f5baeabdb17284d8606331ab7d49 +R 9499438a1fccf595ef80fbd4c31b20b8 U drh -Z cb611c4b71e8fc233ea4e1005f56a9e0 +Z cdff8327924b867b12e561826d5fe961 diff --git a/manifest.uuid b/manifest.uuid index 7417ef4fc1..8b94233ae7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file +b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8d2a891870..2e42b8fc32 100644 --- a/src/where.c +++ b/src/where.c @@ -1612,6 +1612,7 @@ static void constructAutomaticIndex( idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ pPartial = sqlite3ExprAnd(pParse->db, pPartial, sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); @@ -1720,6 +1721,7 @@ static void constructAutomaticIndex( if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(v); sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); @@ -2865,6 +2867,8 @@ static void explainOneScan( if( isSearch ){ zFmt = "PRIMARY KEY"; } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; }else if( flags & WHERE_AUTO_INDEX ){ zFmt = "AUTOMATIC COVERING INDEX"; }else if( flags & WHERE_IDX_ONLY ){ diff --git a/src/whereInt.h b/src/whereInt.h index e9eb8b7dd2..fd4cfdf88a 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -459,3 +459,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ From d05ab6aacf08a1fd7af581cae4b28390ecd5c44c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 13:42:16 +0000 Subject: [PATCH 24/51] Increase the resolution of the second parameter to the likelihood() SQL function (the probability value) so that it can handle probabilities as small as 0.00000001. Formerly, it ran out of precision at 0.001. FossilOrigin-Name: 0f08924fe0c52a85a103f67bee9809e0f8f884b0 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 4 ++-- src/sqliteInt.h | 2 +- src/where.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 28fa8e1387..6d8395af5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. -D 2014-10-25T12:28:25.871 +C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. +D 2014-10-25T13:42:16.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 +F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 +F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d95d0313c447f5baeabdb17284d8606331ab7d49 -R 9499438a1fccf595ef80fbd4c31b20b8 +P b9ad601eab1d7298d369267eb697c7fa1bc16985 +R 06fd31d9e4c300e561fccc4cf28a702b U drh -Z cdff8327924b867b12e561826d5fe961 +Z 628a0c5751868817bd40f1740c611aef diff --git a/manifest.uuid b/manifest.uuid index 8b94233ae7..3804da23b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file +0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index e507ccb810..8fb580b3a1 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -584,7 +584,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -716,7 +716,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to ** likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } #ifndef SQLITE_OMIT_AUTHORIZATION diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b7f992c34d..5114ccccf4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2000,7 +2000,7 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ diff --git a/src/where.c b/src/where.c index 2e42b8fc32..358d508260 100644 --- a/src/where.c +++ b/src/where.c @@ -225,7 +225,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; } From 89ca0b3878bf964c202e1d0c984b39a9eefaafb0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Oct 2014 20:36:28 +0000 Subject: [PATCH 25/51] Further modifications to new code to better handle corrupt databases. FossilOrigin-Name: 1a8cf0a043347772ac54d150d634c32845beee8b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6c0464217b..1a5be4bf57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. -D 2014-10-24T20:57:03.500 +C Further\smodifications\sto\snew\scode\sto\sbetter\shandle\scorrupt\sdatabases. +D 2014-10-25T20:36:28.557 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 +F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc -R c9d1872515b77329bc9fbfe4c8e91668 +P ecc3544e712041736af7c7b4f34864a1f2e30ff7 +R 17dc589efbb3f16f6448c9d9d1ae2e24 U dan -Z 931e363227b1144b3ac19bee87a82036 +Z c615066e82cbd286f5597ce396069fd7 diff --git a/manifest.uuid b/manifest.uuid index 470b2345ea..23643921ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file +1a8cf0a043347772ac54d150d634c32845beee8b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 83ed66414c..46ce39abc7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6099,6 +6099,7 @@ static int pageFreeArray( ){ u8 * const aData = pPg->aData; u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; u8 *pFree = 0; @@ -6106,12 +6107,13 @@ static int pageFreeArray( for(i=0; iaData && pCell=pStart && pCellpEnd ) return 0; }else{ pFree = pCell; szFree += sz; @@ -6854,6 +6856,18 @@ static int balance_nonroot( for(i=0; ipgno; aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; jnFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); + rc = defragmentPage(apNew[0]); + if( rc==SQLITE_OK ){ + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + } }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 61e94c9deb9d0edff1fdaa443fe7668cbfacedb4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Oct 2014 08:02:16 +0000 Subject: [PATCH 26/51] If a free-slot is found within a page, but using that free-slot would fragment the page further and there are already at least 60 fragmented bytes, degragment the page. This matches the behaviour of the trunk. FossilOrigin-Name: 1f80f8c136ac970dcc7fb2337263dc5922e348c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bafcf0fb91..a088218356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\swith\sthis\sbranch. -D 2014-10-27T07:01:04.796 +C If\sa\sfree-slot\sis\sfound\swithin\sa\spage,\sbut\susing\sthat\sfree-slot\swould\sfragment\sthe\spage\sfurther\sand\sthere\sare\salready\sat\sleast\s60\sfragmented\sbytes,\sdegragment\sthe\spage.\sThis\smatches\sthe\sbehaviour\sof\sthe\strunk. +D 2014-10-27T08:02:16.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 +F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a8cf0a043347772ac54d150d634c32845beee8b 0f08924fe0c52a85a103f67bee9809e0f8f884b0 -R 9858b90cbf912e211c65716d59c94e8a +P a13df3013bbac4a0d4fce5cef1376c857508c1c5 +R 05f44d14ab419727640258148977021c U dan -Z daf2b363c619dfc32da3ad287a4e25ec +Z 95c7c43e9818bc003f29ce4926da458f diff --git a/manifest.uuid b/manifest.uuid index 962b2e06bb..92680eaebb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13df3013bbac4a0d4fce5cef1376c857508c1c5 \ No newline at end of file +1f80f8c136ac970dcc7fb2337263dc5922e348c3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 46ce39abc7..9b14dc2b6e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1235,8 +1235,12 @@ static int defragmentPage(MemPage *pPage){ ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. +** +** If a slot of at least nByte bytes is found but cannot be used because +** there are already at least 60 fragmented bytes on the page, return NULL. +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ const int hdr = pPg->hdrOffset; u8 * const aData = pPg->aData; int iAddr; @@ -1255,7 +1259,10 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; + if( aData[hdr+7]>=60 ){ + if( pbDefrag ) *pbDefrag = 1; + return 0; + } /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); @@ -1326,8 +1333,10 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ int rc = SQLITE_OK; - u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + int bDefrag = 0; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; + if( bDefrag ) goto defragment_page; if( pSpace ){ *pIdx = pSpace - data; return SQLITE_OK; @@ -1339,6 +1348,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -6069,7 +6079,7 @@ static int pageInsertArray( for(i=0; i Date: Mon, 27 Oct 2014 11:25:28 +0000 Subject: [PATCH 27/51] Add test file e_wal.test. FossilOrigin-Name: fc6920b5483eeeb06a474ff399a21afa51dc4859 --- manifest | 13 +-- manifest.uuid | 2 +- test/e_wal.test | 229 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 test/e_wal.test diff --git a/manifest b/manifest index 6d8395af5e..6c3a597a05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. -D 2014-10-25T13:42:16.126 +C Add\stest\sfile\se_wal.test. +D 2014-10-27T11:25:28.528 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -466,6 +466,7 @@ F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 +F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1206,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b9ad601eab1d7298d369267eb697c7fa1bc16985 -R 06fd31d9e4c300e561fccc4cf28a702b -U drh -Z 628a0c5751868817bd40f1740c611aef +P 0f08924fe0c52a85a103f67bee9809e0f8f884b0 +R 770a2e0b8a993a6006545c850fbc4dd8 +U dan +Z 70c394a419fcdd8591ee5c3f4c5346c4 diff --git a/manifest.uuid b/manifest.uuid index 3804da23b9..a94336a518 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file +fc6920b5483eeeb06a474ff399a21afa51dc4859 \ No newline at end of file diff --git a/test/e_wal.test b/test/e_wal.test new file mode 100644 index 0000000000..a5e074f49b --- /dev/null +++ b/test/e_wal.test @@ -0,0 +1,229 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_wal + +db close +testvfs oldvfs -iversion 1 + + +# EVIDENCE-OF: R-58297-14483 WAL databases can be created, read, and +# written even if shared memory is unavailable as long as the +# locking_mode is set to EXCLUSIVE before the first attempted access. +# +# EVIDENCE-OF: R-00449-33772 This feature allows WAL databases to be +# created, read, and written by legacy VFSes that lack the "version 2" +# shared-memory methods xShmMap, xShmLock, xShmBarrier, and xShmUnmap on +# the sqlite3_io_methods object. +# +# 1.1: "create" tests. +# 1.2: "read" tests. +# 1.3: "write" tests. +# +# All three done with VFS "oldvfs", which has iVersion==1 and so does +# not support shared memory. +# +sqlite3 db test.db -vfs oldvfs +do_execsql_test 1.1.1 { + PRAGMA journal_mode = WAL; +} {delete} +do_execsql_test 1.1.2 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA journal_mode = WAL; +} {exclusive wal} +do_execsql_test 1.1.3 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} {} +do_test 1.1.4 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.2.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +do_test 1.2.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { SELECT * FROM t1 } +} {1 2} +do_test 1.2.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.3.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { INSERT INTO t1 VALUES(3, 4) } +} {1 {unable to open database file}} +do_test 1.3.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { INSERT INTO t1 VALUES(3, 4) } + execsql { SELECT * FROM t1 } +} {1 2 3 4} +do_test 1.3.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-31969-57825 If EXCLUSIVE locking mode is set prior to +# the first WAL-mode database access, then SQLite never attempts to call +# any of the shared-memory methods and hence no shared-memory wal-index +# is ever created. +# +db close +sqlite3 db test.db +do_execsql_test 2.1.1 { + PRAGMA locking_mode = EXCLUSIVE; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.1.2 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-36328-16367 In that case, the database connection +# remains in EXCLUSIVE mode as long as the journal mode is WAL; attempts +# to change the locking mode using "PRAGMA locking_mode=NORMAL;" are +# no-ops. +# +do_execsql_test 2.2.1 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.2.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +db2 close + +# EVIDENCE-OF: R-63522-46088 The only way to change out of EXCLUSIVE +# locking mode is to first change out of WAL journal mode. +# +do_execsql_test 2.3.1 { + PRAGMA journal_mode = DELETE; + SELECT * FROM t1; +} {delete 1 2 3 4} +do_test 2.3.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +do_execsql_test 2.3.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4} +do_test 2.3.4 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {0 {1 2 3 4}} +db2 close +db close + + +# EVIDENCE-OF: R-57239-11845 If NORMAL locking mode is in effect for the +# first WAL-mode database access, then the shared-memory wal-index is +# created. +# +do_test 3.0 { + sqlite3 db test.db + execsql { PRAGMA journal_mode = WAL } + db close +} {} +do_test 3.1 { + sqlite3 db test.db + execsql { SELECT * FROM t1 } + list [file exists test.db-shm] [file exists test.db-wal] +} {1 1} + +# EVIDENCE-OF: R-13779-07711 As long as exactly one connection is using +# a shared-memory wal-index, the locking mode can be changed freely +# between NORMAL and EXCLUSIVE. +# +do_execsql_test 3.2.1 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA locking_mode = NORMAL; + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(5, 6); +} {exclusive normal exclusive} +do_test 3.2.2 { + sqlite3 db2 test.db + catchsql { SELECT * FROM t1 } db2 +} {1 {database is locked}} + +# EVIDENCE-OF: R-10993-11647 It is only when the shared-memory wal-index +# is omitted, when the locking mode is EXCLUSIVE prior to the first +# WAL-mode database access, that the locking mode is stuck in EXCLUSIVE. +# +do_execsql_test 3.2.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4 5 6} +do_test 3.2.4 { + catchsql { SELECT * FROM t1 } db2 +} {0 {1 2 3 4 5 6}} + +do_catchsql_test 3.2.5 { + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(7, 8); +} {1 {database is locked}} + +db2 close + +# EVIDENCE-OF: R-46197-42811 This means that the underlying VFS must +# support the "version 2" shared-memory. +# +# EVIDENCE-OF: R-55316-21772 If the VFS does not support shared-memory +# methods, then the attempt to open a database that is already in WAL +# mode, or the attempt convert a database into WAL mode, will fail. +# +db close +do_test 3.4.1 { + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +db close +do_test 3.4.2 { + forcedelete test.db2 + sqlite3 db test.db2 -vfs oldvfs + catchsql { PRAGMA journal_mode = WAL } +} {0 delete} +db close + + +# EVIDENCE-OF: R-22428-28959 To prevent older versions of SQLite from +# trying to recover a WAL-mode database (and making matters worse) the +# database file format version numbers (bytes 18 and 19 in the database +# header) are increased from 1 to 2 in WAL mode. +# +reset_db +do_execsql_test 4.1.1 { CREATE TABLE t1(x, y) } +do_test 4.1.2 { hexio_read test.db 18 2 } {0101} +do_execsql_test 4.1.3 { PRAGMA journal_mode = wAL } {wal} +do_test 4.1.4 { hexio_read test.db 18 2 } {0202} + + +# EVIDENCE-OF: R-02535-05811 One can explicitly change out of WAL mode +# using a pragma such as this: PRAGMA journal_mode=DELETE; +# +do_execsql_test 4.2.1 { INSERT INTO t1 VALUES(1, 1); } {} +do_test 4.2.2 { file exists test.db-wal } {1} +do_execsql_test 4.2.3 { PRAGMA journal_mode = delete } {delete} +do_test 4.2.4 { file exists test.db-wal } {0} + +# EVIDENCE-OF: R-60175-02388 Deliberately changing out of WAL mode +# changes the database file format version numbers back to 1 so that +# older versions of SQLite can once again access the database file. +# +do_test 4.3 { hexio_read test.db 18 2 } {0101} + +finish_test From 8a42d1bba15c0ba571014d95b64d4d74759d39fa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 18:21:01 +0000 Subject: [PATCH 28/51] Fix an unused variable in btree.c:allocateSpace(). FossilOrigin-Name: 637246165a14c4808b90d0437e4d43fa5fac659e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 94ec16445a..3991d3d55b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\saimed\sat\sreducing\sthe\snumber\sof\smemcpy()\soperations\srequired\sby\sbalance_nonroot(). -D 2014-10-27T14:26:54.508 +C Fix\san\sunused\svariable\sin\sbtree.c:allocateSpace(). +D 2014-10-27T18:21:01.498 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 +F src/btree.c 812c03daa8be68daf623dd0349ecb18e9f988967 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc6920b5483eeeb06a474ff399a21afa51dc4859 1f80f8c136ac970dcc7fb2337263dc5922e348c3 -R ed2304a6a2e587056109284e9717f063 -U dan -Z 986c15e36469a97fa56839a4b2a55b25 +P face33bea1ba3a6d57780655fa827226b4d2baa9 +R 10f6daecfdf5443eb311db2d487f32a3 +U drh +Z a48d68bafb210b7d63c9203bfcb49102 diff --git a/manifest.uuid b/manifest.uuid index 37f4629c87..28889facf2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -face33bea1ba3a6d57780655fa827226b4d2baa9 \ No newline at end of file +637246165a14c4808b90d0437e4d43fa5fac659e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9b14dc2b6e..06dfd2cd53 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1301,7 +1301,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ int top; /* First byte of cell content area */ int gap; /* First byte of gap between cell pointers and cell content */ int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -1309,8 +1308,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); + assert( nByte < pPage->pBt->usableSize-8 ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; From 98d94211ced2604da384d16978f783b4f166cfdb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 19:39:51 +0000 Subject: [PATCH 29/51] Fix a typo in the macro name of an #ifdef FossilOrigin-Name: 9646a136e69cf2583965dfc9fac5f056af4cdb62 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/ctime.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e658f3a76a..3bfe9532df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption\sfor\sextra\sAPI\sparameter\nvalidation.\s\sEnhance\ssqlite3_stricmp(),\ssqlite3_strnicmp(),\sand\nsqlite3_uri_parameter()\sfor\simproved\sNULL\sparameter\shandling. -D 2014-10-27T18:34:07.031 +C Fix\sa\stypo\sin\sthe\smacro\sname\sof\san\s#ifdef +D 2014-10-27T19:39:51.561 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 -F src/ctime.c dfa83bfebb4201d07b16534acb8a0149592c3a25 +F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee @@ -1207,8 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 637246165a14c4808b90d0437e4d43fa5fac659e 1c220b806d56e163842e17038c3331f71861bd9c -R 78cdca5c720e64c06c4c8ef41bcc31b7 -T +closed 1c220b806d56e163842e17038c3331f71861bd9c +P ffb9d8144bbc35bf3d929e0e13a663668fff0558 +R 517f6fa1cedd5526f59aa045554cf442 U drh -Z 1739c9dd8a9fd2ca84e2afadfbbcea12 +Z d3a8b97f7ef904d82264d3ec79cf2202 diff --git a/manifest.uuid b/manifest.uuid index 2f7fda9cba..70a2bdb89a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffb9d8144bbc35bf3d929e0e13a663668fff0558 \ No newline at end of file +9646a136e69cf2583965dfc9fac5f056af4cdb62 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 36d0c266ee..59dc972d8d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -392,7 +392,7 @@ static const char * const azCompileOpt[] = { int sqlite3_compileoption_used(const char *zOptName){ int i, n; -#ifdef SQLITE_ENABLE_API_ARMORE +#ifdef SQLITE_ENABLE_API_ARMOR if( zOptName==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; From a95d8ca1fad1133a39ab65c60e7b7346f9089749 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 19:42:02 +0000 Subject: [PATCH 30/51] Fix harmless compiler warning in an assert statement. FossilOrigin-Name: d33a1ff3aad0bfabf70a98ac338a68f82074e4fe --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3bfe9532df..c24ca49842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\smacro\sname\sof\san\s#ifdef -D 2014-10-27T19:39:51.561 +C Fix\sharmless\scompiler\swarning\sin\san\sassert\sstatement. +D 2014-10-27T19:42:02.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 812c03daa8be68daf623dd0349ecb18e9f988967 +F src/btree.c 0e3262c36b69bed3c5f9260313286a2e8c060ca5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ffb9d8144bbc35bf3d929e0e13a663668fff0558 -R 517f6fa1cedd5526f59aa045554cf442 -U drh -Z d3a8b97f7ef904d82264d3ec79cf2202 +P 9646a136e69cf2583965dfc9fac5f056af4cdb62 +R c8728e917d07299237654ca84312b584 +U mistachkin +Z 269b8116d90b241ccd2319871bd381eb diff --git a/manifest.uuid b/manifest.uuid index 70a2bdb89a..3bcc889db6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9646a136e69cf2583965dfc9fac5f056af4cdb62 \ No newline at end of file +d33a1ff3aad0bfabf70a98ac338a68f82074e4fe \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 06dfd2cd53..39d1e81a3f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1308,7 +1308,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - assert( nByte < pPage->pBt->usableSize-8 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; From df9c093e2c516c011ec476cf220d477eea70983c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 19:58:29 +0000 Subject: [PATCH 31/51] Fix compilation issue with MSVC due to a misplaced variable declaration. FossilOrigin-Name: 9588b345d09daaa49d24d7fb6cab732e64e5474e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 16 +++++++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c24ca49842..4a6bd095ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\san\sassert\sstatement. -D 2014-10-27T19:42:02.875 +C Fix\scompilation\sissue\swith\sMSVC\sdue\sto\sa\smisplaced\svariable\sdeclaration. +D 2014-10-27T19:58:29.156 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c b8a058131851de1a37801b5587845ee73411c064 +F src/random.c f88232b90e308f58f1f8f10894d7a8a750d6f64d F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9646a136e69cf2583965dfc9fac5f056af4cdb62 -R c8728e917d07299237654ca84312b584 +P d33a1ff3aad0bfabf70a98ac338a68f82074e4fe +R ee98bc34f841b6720302924cb59e4bc2 U mistachkin -Z 269b8116d90b241ccd2319871bd381eb +Z 8119f44960013a2ca52459062580ce07 diff --git a/manifest.uuid b/manifest.uuid index 3bcc889db6..f68a004aea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d33a1ff3aad0bfabf70a98ac338a68f82074e4fe \ No newline at end of file +9588b345d09daaa49d24d7fb6cab732e64e5474e \ No newline at end of file diff --git a/src/random.c b/src/random.c index c4241362de..bd109e71b3 100644 --- a/src/random.c +++ b/src/random.c @@ -34,10 +34,6 @@ void sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return; -#endif - /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, ** we have to locate the state vector at run-time. In the more common @@ -52,13 +48,23 @@ void sqlite3_randomness(int N, void *pBuf){ #endif #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); + sqlite3_mutex *mutex; +#endif + +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); sqlite3_mutex_enter(mutex); #endif if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; +#if SQLITE_THREADSAFE sqlite3_mutex_leave(mutex); +#endif return; } From d61a18a9f1493ed6ee2b6d12d7b4ca4621cc894b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 20:14:02 +0000 Subject: [PATCH 32/51] Remove a small amount of unnecessary #ifdeffery from random.c. FossilOrigin-Name: 2b9340c8684bc382391e02813e960b3166f24daa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/random.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4a6bd095ba..31157b57d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissue\swith\sMSVC\sdue\sto\sa\smisplaced\svariable\sdeclaration. -D 2014-10-27T19:58:29.156 +C Remove\sa\ssmall\samount\sof\sunnecessary\s#ifdeffery\sfrom\srandom.c. +D 2014-10-27T20:14:02.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c f88232b90e308f58f1f8f10894d7a8a750d6f64d +F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d33a1ff3aad0bfabf70a98ac338a68f82074e4fe -R ee98bc34f841b6720302924cb59e4bc2 -U mistachkin -Z 8119f44960013a2ca52459062580ce07 +P 9588b345d09daaa49d24d7fb6cab732e64e5474e +R e26f0edef3def3003fae07f43e971e87 +U drh +Z 3e6e6f10ca8455ff968ead5f51a9fff2 diff --git a/manifest.uuid b/manifest.uuid index f68a004aea..11afc870ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9588b345d09daaa49d24d7fb6cab732e64e5474e \ No newline at end of file +2b9340c8684bc382391e02813e960b3166f24daa \ No newline at end of file diff --git a/src/random.c b/src/random.c index bd109e71b3..179d01bef2 100644 --- a/src/random.c +++ b/src/random.c @@ -57,14 +57,12 @@ void sqlite3_randomness(int N, void *pBuf){ #if SQLITE_THREADSAFE mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); - sqlite3_mutex_enter(mutex); #endif + sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; -#if SQLITE_THREADSAFE sqlite3_mutex_leave(mutex); -#endif return; } From 2d8ad51c5b53644aa40ab276a7c7342da920d465 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 22:06:21 +0000 Subject: [PATCH 33/51] Add special handling for static mutexes in sqlite3_mutex_alloc() when automatic calls to sqlite3_initialize() are enabled. FossilOrigin-Name: 7857d27caa845e5629d94c2e66587dc89016daca --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 31157b57d0..ed6b523e7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssmall\samount\sof\sunnecessary\s#ifdeffery\sfrom\srandom.c. -D 2014-10-27T20:14:02.021 +C Add\sspecial\shandling\sfor\sstatic\smutexes\sin\ssqlite3_mutex_alloc()\swhen\sautomatic\scalls\sto\ssqlite3_initialize()\sare\senabled. +D 2014-10-27T22:06:21.571 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -202,7 +202,7 @@ F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 -F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c +F src/mutex.c 19bf9acba69ca2f367c3761080f8a9f0cf4670a8 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9588b345d09daaa49d24d7fb6cab732e64e5474e -R e26f0edef3def3003fae07f43e971e87 -U drh -Z 3e6e6f10ca8455ff968ead5f51a9fff2 +P 2b9340c8684bc382391e02813e960b3166f24daa +R f7a6b10545dea42b1d622484e6364982 +U mistachkin +Z 6b079baac5628ec298fadd129a245092 diff --git a/manifest.uuid b/manifest.uuid index 11afc870ef..7088378432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b9340c8684bc382391e02813e960b3166f24daa \ No newline at end of file +7857d27caa845e5629d94c2e66587dc89016daca \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index bad5a7c113..2b45036289 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -82,6 +82,7 @@ int sqlite3MutexEnd(void){ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } From eefaf448db0ed620f87af06a49c1bc5fbfdb9282 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 00:56:18 +0000 Subject: [PATCH 34/51] Correct the documentation on the maximum size of a scratch allocation. FossilOrigin-Name: 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 1 + src/sqlite.h.in | 12 ++++++++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ed6b523e7c..2a931c6cb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sspecial\shandling\sfor\sstatic\smutexes\sin\ssqlite3_mutex_alloc()\swhen\sautomatic\scalls\sto\ssqlite3_initialize()\sare\senabled. -D 2014-10-27T22:06:21.571 +C Correct\sthe\sdocumentation\son\sthe\smaximum\ssize\sof\sa\sscratch\sallocation. +D 2014-10-28T00:56:18.775 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 0e3262c36b69bed3c5f9260313286a2e8c060ca5 +F src/btree.c 5189881ca403938c5ceddde496b984fef9f40c5a F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 +F src/sqlite.h.in a9f2e5a0e2472c8c7819f3a16074c14b9376608f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2b9340c8684bc382391e02813e960b3166f24daa -R f7a6b10545dea42b1d622484e6364982 -U mistachkin -Z 6b079baac5628ec298fadd129a245092 +P 7857d27caa845e5629d94c2e66587dc89016daca +R cc3ad0fbd146620828bc67546f9cb55d +U drh +Z 46e16e5efd8acf1525fc0288e922960d diff --git a/manifest.uuid b/manifest.uuid index 7088378432..bc344729d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7857d27caa845e5629d94c2e66587dc89016daca \ No newline at end of file +30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 39d1e81a3f..22b168d9e4 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6649,6 +6649,7 @@ static int balance_nonroot( nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ + assert( szScratch<=16896 || szScratch<=6*pBt->pageSize ); apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f1d4e406e8..8cfe61ee1a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1539,10 +1539,14 @@ struct sqlite3_mem_methods { ** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than two scratch buffers per thread. So -** N should be set to twice the expected maximum number of threads. -** ^SQLite will never require a scratch buffer that is more than 6 -** times the database page size. ^If SQLite needs needs additional +** ^SQLite will not use more than two scratch buffers per thread and not +** more than one scratch buffer per thread when not performing +** a [checkpoint] in [WAL mode]. +** ^SQLite will never request a scratch buffer that is more than 6 +** times the database page size, except when performing a [checkpoint] +** in [WAL mode] when the scratch buffer request size is a small fraction +** of the size of the WAL file. +** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed. ** From 916cd23b2ca6aa4716b7b407abea6d870ea4585c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 12:35:28 +0000 Subject: [PATCH 35/51] Bump the version number to 3.8.8 FossilOrigin-Name: 1a7e711ed10860c7985e84b97cdfff748d940b9f --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 4351a7e3a3..d7d8e42315 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7 +3.8.8 diff --git a/configure b/configure index 9b8266d812..c0f751db62 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7' -PACKAGE_STRING='sqlite 3.8.7' +PACKAGE_VERSION='3.8.8' +PACKAGE_STRING='sqlite 3.8.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7:";; + short | recursive ) echo "Configuration of sqlite 3.8.8:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7 +sqlite configure 3.8.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7, which was +It was created by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7, which was +This file was extended by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7 +sqlite config.status 3.8.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 2a931c6cb3..27680c927a 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Correct\sthe\sdocumentation\son\sthe\smaximum\ssize\sof\sa\sscratch\sallocation. -D 2014-10-28T00:56:18.775 +C Bump\sthe\sversion\snumber\sto\s3.8.8 +D 2014-10-28T12:35:28.069 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 53a0b870e7f16d3b06623c31d233a304c163a6af +F VERSION d846487aff892625eb8e75960234e7285f0462fe F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure ad59a5f48b3c59a92b5506040a22fbe3f733a9d8 x +F configure 4343c810cc772571210af75d1a8f7c2eb711d75a x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7857d27caa845e5629d94c2e66587dc89016daca -R cc3ad0fbd146620828bc67546f9cb55d +P 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 +R 241b510f33643f46040372d1284a7d7b U drh -Z 46e16e5efd8acf1525fc0288e922960d +Z b060cb081e36e4268c2624adfeebfbc8 diff --git a/manifest.uuid b/manifest.uuid index bc344729d5..4cdb1c73e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 \ No newline at end of file +1a7e711ed10860c7985e84b97cdfff748d940b9f \ No newline at end of file From c3ef4fa88a370dc6835d6dc4c3875d9577ae34f5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 15:58:50 +0000 Subject: [PATCH 36/51] Trivial simplification to the automatic index logic. FossilOrigin-Name: 23073a053931de324323f631a6613086786af411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 27680c927a..4d085dbcbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bump\sthe\sversion\snumber\sto\s3.8.8 -D 2014-10-28T12:35:28.069 +C Trivial\ssimplification\sto\sthe\sautomatic\sindex\slogic. +D 2014-10-28T15:58:50.607 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 +F src/where.c 5665df88cbd2b38eb72b4b94c8892c8afb360181 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 -R 241b510f33643f46040372d1284a7d7b +P 1a7e711ed10860c7985e84b97cdfff748d940b9f +R b0011529707271df74d0fa481aad6091 U drh -Z b060cb081e36e4268c2624adfeebfbc8 +Z ab603a6740e2096395006a3b0e96e405 diff --git a/manifest.uuid b/manifest.uuid index 4cdb1c73e1..39c8a652a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a7e711ed10860c7985e84b97cdfff748d940b9f \ No newline at end of file +23073a053931de324323f631a6613086786af411 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 358d508260..feccf2d11d 100644 --- a/src/where.c +++ b/src/where.c @@ -1651,7 +1651,7 @@ static void constructAutomaticIndex( ** if they go out of sync. */ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & MASKBIT(BMS-1) ){ nKeyCol += pTable->nCol - BMS + 1; } - pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); From 635e57fc4893060c6c31efa3bcd936a9720212c7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 16:19:18 +0000 Subject: [PATCH 37/51] Fix a faulty assert() in the DELETE code generator. FossilOrigin-Name: 95f8ebdbf87326f23cd38e561ac5632b5367a449 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/delete.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4d085dbcbc..a6054c1a60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Trivial\ssimplification\sto\sthe\sautomatic\sindex\slogic. -D 2014-10-28T15:58:50.607 +C Fix\sa\sfaulty\sassert()\sin\sthe\sDELETE\scode\sgenerator. +D 2014-10-28T16:19:18.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 -F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f +F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a7e711ed10860c7985e84b97cdfff748d940b9f -R b0011529707271df74d0fa481aad6091 +P 23073a053931de324323f631a6613086786af411 +R 5f4e1ea821e16c5044076b23f1aebe8a U drh -Z ab603a6740e2096395006a3b0e96e405 +Z 478016a63ef8d014d3b6d95929bf74df diff --git a/manifest.uuid b/manifest.uuid index 39c8a652a5..49ec9c5970 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23073a053931de324323f631a6613086786af411 \ No newline at end of file +95f8ebdbf87326f23cd38e561ac5632b5367a449 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index b97407400b..d81dd3f6b4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -481,7 +481,7 @@ void sqlite3DeleteFrom( assert( nKey==nPk ); /* OP_Found will use an unpacked key */ assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } From dd8c460081ef024abbb925029a18adb5deea3522 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 16:50:10 +0000 Subject: [PATCH 38/51] =?UTF-8?q?Fix=20a=20crash=20that=20could=20occur=20?= =?UTF-8?q?if=20the=20WHERE=20clause=20of=20an=20UPDATE=20statement=20on?= =?UTF-8?q?=20a=20view=20that=20does=20not=20feature=20a=20column=20named?= =?UTF-8?q?=20"rowid"=20contains=20a=20term=20such=20as=20"rowid=3D=3F".?= FossilOrigin-Name: 8523670d50004f3112b7871f11c8b8b02aab96ab --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/update.c | 4 ++-- test/trigger9.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a6054c1a60..8f4042bf5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sDELETE\scode\sgenerator. -D 2014-10-28T16:19:18.944 +C Fix\sa\scrash\sthat\scould\soccur\sif\sthe\sWHERE\sclause\sof\san\sUPDATE\sstatement\son\sa\sview\sthat\sdoes\snot\sfeature\sa\scolumn\snamed\s"rowid"\scontains\sa\sterm\ssuch\sas\s"rowid=?". +D 2014-10-28T16:50:10.527 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f -F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 +F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a @@ -1051,7 +1051,7 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test b39e6dee1debe0ff9c2ef66326668f149f07c9c4 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 +F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 23073a053931de324323f631a6613086786af411 -R 5f4e1ea821e16c5044076b23f1aebe8a -U drh -Z 478016a63ef8d014d3b6d95929bf74df +P 95f8ebdbf87326f23cd38e561ac5632b5367a449 +R c4fade85ca4f95e680190bc9e417c1ff +U dan +Z 12d8cc683b13ea652b467efe210b5438 diff --git a/manifest.uuid b/manifest.uuid index 49ec9c5970..110e17e040 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95f8ebdbf87326f23cd38e561ac5632b5367a449 \ No newline at end of file +8523670d50004f3112b7871f11c8b8b02aab96ab \ No newline at end of file diff --git a/src/update.c b/src/update.c index f781a60ccd..3af4017f1b 100644 --- a/src/update.c +++ b/src/update.c @@ -431,8 +431,8 @@ void sqlite3Update( /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } diff --git a/test/trigger9.test b/test/trigger9.test index f56c8acbc5..326fa63d4c 100644 --- a/test/trigger9.test +++ b/test/trigger9.test @@ -32,6 +32,7 @@ ifcapable {!trigger} { finish_test return } +set ::testprefix trigger9 proc has_rowdata {sql} { expr {[lsearch [execsql "explain $sql"] RowData]>=0} @@ -220,4 +221,36 @@ ifcapable compound { } {2} } +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + CREATE VIEW v1 AS SELECT a, b FROM t1; + + CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'); + END; + + CREATE TRIGGER tr3 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; +} + +do_execsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; +} {} + +do_execsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; +} {} + + + + finish_test From c3da667b25cd513f3552652e783990734a200992 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 18:24:16 +0000 Subject: [PATCH 39/51] Modify the documentation for sqlite3_changes() to make it more testable. Add tests and minor fixes for the same. FossilOrigin-Name: 41cdd0c422d61533a94870cb5ad094682956d472 --- manifest | 19 +- manifest.uuid | 2 +- src/sqlite.h.in | 78 ++++---- src/vdbe.c | 1 + src/vdbeInt.h | 3 +- src/vdbeaux.c | 6 + test/e_changes.test | 441 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 499 insertions(+), 51 deletions(-) create mode 100644 test/e_changes.test diff --git a/manifest b/manifest index 8f4042bf5b..380d7ccd48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scould\soccur\sif\sthe\sWHERE\sclause\sof\san\sUPDATE\sstatement\son\sa\sview\sthat\sdoes\snot\sfeature\sa\scolumn\snamed\s"rowid"\scontains\sa\sterm\ssuch\sas\s"rowid=?". -D 2014-10-28T16:50:10.527 +C Modify\sthe\sdocumentation\sfor\ssqlite3_changes()\sto\smake\sit\smore\stestable.\sAdd\stests\sand\sminor\sfixes\sfor\sthe\ssame. +D 2014-10-28T18:24:16.387 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in a9f2e5a0e2472c8c7819f3a16074c14b9376608f +F src/sqlite.h.in 1c5624f8b21cc8261a8b048033815581347b375f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 +F src/vdbe.c 1b7e8ccaca2a23ae2804568f34b7c645adfd332d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 +F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 -F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 +F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -451,6 +451,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 @@ -1207,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 95f8ebdbf87326f23cd38e561ac5632b5367a449 -R c4fade85ca4f95e680190bc9e417c1ff +P 8523670d50004f3112b7871f11c8b8b02aab96ab +R 8192ca2d13d614d67e4c391f6685b170 U dan -Z 12d8cc683b13ea652b467efe210b5438 +Z 02a949e02b7399a00fe453126f032ba0 diff --git a/manifest.uuid b/manifest.uuid index 110e17e040..1803fe3792 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8523670d50004f3112b7871f11c8b8b02aab96ab \ No newline at end of file +41cdd0c422d61533a94870cb5ad094682956d472 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8cfe61ee1a..bb5d90254d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1874,47 +1874,45 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified ** -** ^This function returns the number of database rows that were changed -** or inserted or deleted by the most recently completed SQL statement -** on the [database connection] specified by the first parameter. -** ^(Only changes that are directly specified by the [INSERT], [UPDATE], -** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted.)^ Use the -** [sqlite3_total_changes()] function to find the total number of changes -** including changes caused by triggers and foreign key actions. +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. ** -** ^Changes to a view that are simulated by an [INSTEAD OF trigger] -** are not counted. Only real table changes are counted. +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. ** -** ^(A "row change" is a change to a single row of a single table -** caused by an INSERT, DELETE, or UPDATE statement. Rows that -** are changed as side effects of [REPLACE] constraint resolution, -** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes.)^ -** -** A "trigger context" is a scope of execution that begins and -** ends with the script of a [CREATE TRIGGER | trigger]. -** Most SQL statements are -** evaluated outside of any trigger. This is the "top level" -** trigger context. If a trigger fires from the top level, a -** new trigger context is entered for the duration of that one -** trigger. Subtriggers create subcontexts for their duration. -** -** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does -** not create a new trigger context. -** -** ^This function returns the number of direct row changes in the -** most recent INSERT, UPDATE, or DELETE statement within the same -** trigger context. -** -** ^Thus, when called from the top level, this function returns the -** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. ^(Within the body of a trigger, -** the sqlite3_changes() interface can be called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the same trigger. -** However, the number returned does not include changes -** caused by subtriggers since those have their own context.)^ +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
    +**
  • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
  • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
+** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. @@ -1934,7 +1932,7 @@ int sqlite3_changes(sqlite3*); ** from all [CREATE TRIGGER | trigger] contexts and changes made by ** [foreign key actions]. However, ** the count does not include changes used to implement [REPLACE] constraints, -** do rollbacks or ABORT processing, or [DROP TABLE] processing. The +** rollbacks or [DROP TABLE] commands. The ** count does not include rows of views that fire an [INSTEAD OF trigger], ** though if the INSTEAD OF trigger makes changes of its own, those changes ** are counted.)^ diff --git a/src/vdbe.c b/src/vdbe.c index 0f9f45c456..d256c6b770 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5423,6 +5423,7 @@ case OP_Program: { /* jump */ pFrame->pParent = p->pFrame; pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; p->nChange = 0; p->pFrame = pFrame; p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index bb504d64a1..623c5fdde8 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -144,7 +144,8 @@ struct VdbeFrame { int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - int nChange; /* Statement changes (Vdbe.nChanges) */ + int nChange; /* Statement changes (Vdbe.nChange) */ + int nDbChange; /* Value of db->nChange */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c0018bb71c..7dfb64130d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1772,6 +1772,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->nCursor = pFrame->nCursor; v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; return pFrame->pc; } @@ -2339,6 +2340,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } } @@ -2379,6 +2381,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; @@ -2387,6 +2390,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } }else{ sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2398,6 +2402,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -2418,6 +2423,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } diff --git a/test/e_changes.test b/test/e_changes.test new file mode 100644 index 0000000000..a77e22a2ee --- /dev/null +++ b/test/e_changes.test @@ -0,0 +1,441 @@ +# 2011 October 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_changes + +# Like [do_execsql_test], except it appends the value returned by +# [db changes] to the result of executing the SQL script. +# +proc do_changes_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db changes\]" $res + ] +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-15996-49369 This function returns the number of rows +# modified, inserted or deleted by the most recently completed INSERT, +# UPDATE or DELETE statement on the database connection specified by the +# only parameter. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t2(y); +} +foreach {tn schema} { + 1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b); + } + 2 { + CREATE TABLE t1(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } +} { + reset_db + execsql $schema + + # Insert 1 row. + do_changes_test 1.$tn.1 { INSERT INTO t1 VALUES(0, 0) } 1 + + # Insert 10 rows. + do_changes_test 1.$tn.2 { + WITH rows(i, j) AS ( + SELECT 1, 1 UNION ALL SELECT i+1, j+i FROM rows WHERE i<10 + ) + INSERT INTO t1 SELECT * FROM rows + } 10 + + # Modify 5 rows. + do_changes_test 1.$tn.3 { + UPDATE t1 SET b=b+1 WHERE a<5; + } 5 + + # Delete 4 rows + do_changes_test 1.$tn.4 { + DELETE FROM t1 WHERE a>6 + } 4 + + # Check the "on the database connecton specified" part of hte + # requirement - changes made by other connections do not show up in + # the return value of sqlite3_changes(). + do_test 1.$tn.5 { + sqlite3 db2 test.db + execsql { INSERT INTO t1 VALUES(-1, -1) } db2 + db2 changes + } 1 + do_test 1.$tn.6 { + db changes + } 4 + db2 close + + # Test that statements that modify no rows because they hit UNIQUE + # constraints set the sqlite3_changes() value to 0. Regardless of + # whether or not they are executed inside an explicit transaction. + # + # 1.$tn.8-9: outside of a transaction + # 1.$tn.10-12: inside a transaction + # + do_changes_test 1.$tn.7 { + CREATE UNIQUE INDEX i2 ON t1(a); + } 4 + do_catchsql_test 1.$tn.8 { + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.9 { db changes } 0 + do_catchsql_test 1.$tn.10 { + BEGIN; + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.11 { db changes } 0 + do_changes_test 1.$tn.12 COMMIT 0 + +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-44877-05564 Executing any other type of SQL statement +# does not modify the value returned by this function. +# +reset_db +do_changes_test 2.1 { CREATE TABLE t1(x) } 0 +do_changes_test 2.2 { + WITH d(y) AS (SELECT 1 UNION ALL SELECT y+1 FROM d WHERE y<47) + INSERT INTO t1 SELECT y FROM d; +} 47 + +# The statement above set changes() to 47. Check that none of the following +# modify this. +do_changes_test 2.3 { SELECT count(x) FROM t1 } {47 47} +do_changes_test 2.4 { DROP TABLE t1 } 47 +do_changes_test 2.5 { CREATE TABLE t1(x) } 47 +do_changes_test 2.6 { ALTER TABLE t1 ADD COLUMN b } 47 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-53938-27527 Only changes made directly by the INSERT, +# UPDATE or DELETE statement are considered - auxiliary changes caused +# by triggers, foreign key actions or REPLACE constraint resolution are +# not counted. +# +# 3.1.*: triggers +# 3.2.*: foreign key actions +# 3.3.*: replace constraints +# +reset_db +do_execsql_test 3.1.0 { + CREATE TABLE log(x); + CREATE TABLE p1(one PRIMARY KEY, two); + + CREATE TRIGGER tr_ai AFTER INSERT ON p1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER tr_bd BEFORE DELETE ON p1 BEGIN + INSERT INTO log VALUES('delete'); + END; + CREATE TRIGGER tr_au AFTER UPDATE ON p1 BEGIN + INSERT INTO log VALUES('update'); + END; + +} + +do_changes_test 3.1.1 { + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); +} 3 +do_changes_test 3.1.2 { + UPDATE p1 SET two = two||two; +} 3 +do_changes_test 3.1.3 { + DELETE FROM p1 WHERE one IN ('a', 'c'); +} 2 +do_execsql_test 3.1.4 { + -- None of the inserts on table log were counted. + SELECT count(*) FROM log +} 8 + +do_execsql_test 3.2.0 { + DELETE FROM p1; + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); + + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + INSERT INTO c1 VALUES('a', 'aaa'); + INSERT INTO c2 VALUES('b', 'bbb'); + INSERT INTO c3 VALUES('c', 'ccc'); + + INSERT INTO p1 VALUES('d', 'D'), ('e', 'E'), ('f', 'F'); + CREATE TABLE c4(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c5(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + CREATE TABLE c6(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + INSERT INTO c4 VALUES('d', 'aaa'); + INSERT INTO c5 VALUES('e', 'bbb'); + INSERT INTO c6 VALUES('f', 'ccc'); + + PRAGMA foreign_keys = ON; +} + +do_changes_test 3.2.1 { DELETE FROM p1 WHERE one = 'a' } 1 +do_changes_test 3.2.2 { DELETE FROM p1 WHERE one = 'b' } 1 +do_changes_test 3.2.3 { DELETE FROM p1 WHERE one = 'c' } 1 +do_execsql_test 3.2.4 { + SELECT * FROM c1; + SELECT * FROM c2; + SELECT * FROM c3; +} {{} aaa {} bbb} + +do_changes_test 3.2.5 { UPDATE p1 SET one = 'g' WHERE one = 'd' } 1 +do_changes_test 3.2.6 { UPDATE p1 SET one = 'h' WHERE one = 'e' } 1 +do_changes_test 3.2.7 { UPDATE p1 SET one = 'i' WHERE one = 'f' } 1 +do_execsql_test 3.2.8 { + SELECT * FROM c4; + SELECT * FROM c5; + SELECT * FROM c6; +} {{} aaa {} bbb i ccc} + +do_execsql_test 3.3.0 { + CREATE TABLE r1(a UNIQUE, b UNIQUE); + INSERT INTO r1 VALUES('i', 'i'); + INSERT INTO r1 VALUES('ii', 'ii'); + INSERT INTO r1 VALUES('iii', 'iii'); + INSERT INTO r1 VALUES('iv', 'iv'); + INSERT INTO r1 VALUES('v', 'v'); + INSERT INTO r1 VALUES('vi', 'vi'); + INSERT INTO r1 VALUES('vii', 'vii'); +} + +do_changes_test 3.3.1 { INSERT OR REPLACE INTO r1 VALUES('i', 1) } 1 +do_changes_test 3.3.2 { INSERT OR REPLACE INTO r1 VALUES('iv', 'v') } 1 +do_changes_test 3.3.3 { UPDATE OR REPLACE r1 SET b='v' WHERE a='iii' } 1 +do_changes_test 3.3.4 { UPDATE OR REPLACE r1 SET b='vi',a='vii' WHERE a='ii' } 1 +do_execsql_test 3.3.5 { + SELECT * FROM r1 ORDER BY a; +} {i 1 iii v vii vi} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-09813-48563 The value returned by sqlite3_changes() +# immediately after an INSERT, UPDATE or DELETE statement run on a view +# is always zero. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE log(log); + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER v1_i INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER v1_u INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'), ('update'); + END; + CREATE TRIGGER v1_d INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'), ('delete'), ('delete'); + END; +} + +do_changes_test 4.2.1 { INSERT INTO t1 SELECT * FROM t1 } 3 +do_changes_test 4.2.2 { INSERT INTO v1 VALUES(1, 2) } 0 + +do_changes_test 4.3.1 { INSERT INTO t1 SELECT * FROM t1 } 6 +do_changes_test 4.3.2 { UPDATE v1 SET y='xyz' WHERE x=1 } 0 + +do_changes_test 4.4.1 { INSERT INTO t1 SELECT * FROM t1 } 12 +do_changes_test 4.4.2 { DELETE FROM v1 WHERE x=5 } 0 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-32918-61474 Before entering a trigger program the value +# returned by sqlite3_changes() function is saved. After the trigger +# program has finished, the original value is restored. +# +reset_db +db func my_changes my_changes +set ::changes [list] +proc my_changes {x} { + set res [db changes] + lappend ::changes $x $res + return $res +} + +do_execsql_test 5.1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(x); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, NULL); + INSERT INTO t1 VALUES(3, NULL); + CREATE TRIGGER AFTER UPDATE ON t1 BEGIN + INSERT INTO t2 VALUES('a'), ('b'), ('c'); + SELECT my_changes('trigger'); + END; +} + +do_execsql_test 5.1.1 { + INSERT INTO t2 VALUES('a'), ('b'); + UPDATE t1 SET b = my_changes('update'); + SELECT * FROM t1; +} {1 2 2 2 3 2} + +# Value is being restored to "2" when the trigger program exits. +do_test 5.1.2 { + set ::changes +} {update 2 trigger 3 update 2 trigger 3 update 2 trigger 3} + + +reset_db +do_execsql_test 5.2.0 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 0); + INSERT INTO t1 VALUES(2, 0); + INSERT INTO t1 VALUES(3, 0); + CREATE TRIGGER t1_a_u AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES(old.b || ' -> ' || new.b || ' c = ' || changes() ); + END; + CREATE TABLE t2(a); + INSERT INTO t2 VALUES(1), (2), (3); + UPDATE t1 SET b = changes(); +} +do_execsql_test 5.2.1 { + SELECT * FROM t1; +} {1 3 2 3 3 3} +do_execsql_test 5.2.2 { + SELECT * FROM log; +} {{0 -> 3 c = 3} {0 -> 3 c = 3} {0 -> 3 c = 3}} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-17146-37073 Within a trigger program each INSERT, +# UPDATE and DELETE statement sets the value returned by +# sqlite3_changes() upon completion as normal. Of course, this value +# will not include any changes performed by sub-triggers, as the +# sqlite3_changes() value will be saved and restored after each +# sub-trigger has run. +reset_db +do_execsql_test 6.0 { + + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE t3(a, b); + CREATE TABLE log(x); + + CREATE TRIGGER t1_i BEFORE INSERT ON t1 BEGIN + INSERT INTO t2 VALUES(new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_i AFTER INSERT ON t2 BEGIN + INSERT INTO t3 VALUES(new.a, new.b), (new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_u AFTER UPDATE ON t1 BEGIN + UPDATE t2 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_u BEFORE UPDATE ON t2 BEGIN + UPDATE t3 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_d AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_d BEFORE DELETE ON t2 BEGIN + DELETE FROM t3 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t3->' || changes()); + END; +} + +do_changes_test 6.1 { + INSERT INTO t1 VALUES('+', 'o'); + SELECT * FROM log; +} {t3->3 t3->3 t2->2 1} + +do_changes_test 6.2 { + DELETE FROM log; + UPDATE t1 SET b='*'; + SELECT * FROM log; +} {t3->6 t3->6 t2->2 1} + +do_changes_test 6.3 { + DELETE FROM log; + DELETE FROM t1; + SELECT * FROM log; +} {t3->6 t3->0 t2->2 1} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-43399-09409 This means that if the changes() SQL +# function (or similar) is used by the first INSERT, UPDATE or DELETE +# statement within a trigger, it returns the value as set when the +# calling statement began executing. +# +# EVIDENCE-OF: R-53215-27584 If it is used by the second or subsequent +# such statement within a trigger program, the value returned reflects +# the number of rows modified by the previous INSERT, UPDATE or DELETE +# statement within the same trigger. +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE q1(t); + CREATE TABLE q2(u, v); + CREATE TABLE q3(w); + + CREATE TRIGGER q2_insert BEFORE INSERT ON q2 BEGIN + + /* changes() returns value from previous I/U/D in callers context */ + INSERT INTO q1 VALUES('1:' || changes()); + + /* changes() returns value of previous I/U/D in this context */ + INSERT INTO q3 VALUES(changes()), (2), (3); + INSERT INTO q1 VALUES('2:' || changes()); + INSERT INTO q3 VALUES(changes() + 3), (changes()+4); + SELECT 'this does not affect things!'; + INSERT INTO q1 VALUES('3:' || changes()); + UPDATE q3 SET w = w+10 WHERE w%2; + INSERT INTO q1 VALUES('4:' || changes()); + DELETE FROM q3; + INSERT INTO q1 VALUES('5:' || changes()); + END; +} + +do_execsql_test 7.2 { + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:0 2:3 3:2 4:3 5:5 +} + +do_execsql_test 7.3 { + DELETE FROM q1; + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:5 2:3 3:2 4:3 5:5 +} + + + +finish_test From 4f41b7dec5a99cc72148cf6d0dd66b890cf82619 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 20:35:18 +0000 Subject: [PATCH 40/51] Update the documentation on the sqlite3_randomness() interface to conform to enhancements associated with the SQLITE_ENABLE_API_ARMOR change. FossilOrigin-Name: 96e9917c350dfe2069b87860bbb961424ff1105a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 380d7ccd48..4c55bebcf9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sdocumentation\sfor\ssqlite3_changes()\sto\smake\sit\smore\stestable.\sAdd\stests\sand\sminor\sfixes\sfor\sthe\ssame. -D 2014-10-28T18:24:16.387 +C Update\sthe\sdocumentation\son\sthe\ssqlite3_randomness()\sinterface\sto\sconform\nto\senhancements\sassociated\swith\sthe\sSQLITE_ENABLE_API_ARMOR\schange. +D 2014-10-28T20:35:18.499 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 1c5624f8b21cc8261a8b048033815581347b375f +F src/sqlite.h.in 9c8090268644fba8d2b5b594b9d577b3da23905c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -1208,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8523670d50004f3112b7871f11c8b8b02aab96ab -R 8192ca2d13d614d67e4c391f6685b170 -U dan -Z 02a949e02b7399a00fe453126f032ba0 +P 41cdd0c422d61533a94870cb5ad094682956d472 +R d8b973f48f438632851d26e03df40631 +U drh +Z d1783aa6143d26d5a0a1c09bffad4d99 diff --git a/manifest.uuid b/manifest.uuid index 1803fe3792..6cff793b96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41cdd0c422d61533a94870cb5ad094682956d472 \ No newline at end of file +96e9917c350dfe2069b87860bbb961424ff1105a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bb5d90254d..20835794e8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2417,13 +2417,14 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. -** ^If N is less than one, then P can be a NULL pointer. +** ^The P parameter can be a NULL pointer. ** ** ^If this routine has not been previously called or if the previous -** call had N less than one, then the PRNG is seeded using randomness -** obtained from the xRandomness method of the default [sqlite3_vfs] object. -** ^If the previous call to this routine had an N of 1 or more then -** the pseudo-randomness is generated +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ From aa55563d7eed0ba2e6ab0afd5625b00e30931b1d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 20:49:59 +0000 Subject: [PATCH 41/51] Add new test file e_totalchanges.test, containing tests of the sqlite3_total_changes() interface. FossilOrigin-Name: f84af4adcc34d7a4c72027bf5b038a1a45a4c307 --- manifest | 15 +-- manifest.uuid | 2 +- src/sqlite.h.in | 25 ++--- test/e_totalchanges.test | 213 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 test/e_totalchanges.test diff --git a/manifest b/manifest index 4c55bebcf9..cbbdd001c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\son\sthe\ssqlite3_randomness()\sinterface\sto\sconform\nto\senhancements\sassociated\swith\sthe\sSQLITE_ENABLE_API_ARMOR\schange. -D 2014-10-28T20:35:18.499 +C Add\snew\stest\sfile\se_totalchanges.test,\scontaining\stests\sof\sthe\ssqlite3_total_changes()\sinterface. +D 2014-10-28T20:49:59.213 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 9c8090268644fba8d2b5b594b9d577b3da23905c +F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -464,6 +464,7 @@ F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f +F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 @@ -1208,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 41cdd0c422d61533a94870cb5ad094682956d472 -R d8b973f48f438632851d26e03df40631 -U drh -Z d1783aa6143d26d5a0a1c09bffad4d99 +P 96e9917c350dfe2069b87860bbb961424ff1105a +R ff91cb59544b367ee28dfb499ecb1ca4 +U dan +Z ba0579c9ba051395a01d0b2f40d05365 diff --git a/manifest.uuid b/manifest.uuid index 6cff793b96..6aa58547c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e9917c350dfe2069b87860bbb961424ff1105a \ No newline at end of file +f84af4adcc34d7a4c72027bf5b038a1a45a4c307 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 20835794e8..b4081f2a02 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1926,20 +1926,17 @@ int sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified ** -** ^This function returns the number of row changes caused by [INSERT], -** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** ^(The count returned by sqlite3_total_changes() includes all changes -** from all [CREATE TRIGGER | trigger] contexts and changes made by -** [foreign key actions]. However, -** the count does not include changes used to implement [REPLACE] constraints, -** rollbacks or [DROP TABLE] commands. The -** count does not include rows of views that fire an [INSTEAD OF trigger], -** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted.)^ -** ^The sqlite3_total_changes() function counts the changes as soon as -** the statement that makes them is completed (when the statement handle -** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). -** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** diff --git a/test/e_totalchanges.test b/test/e_totalchanges.test new file mode 100644 index 0000000000..ee163c914f --- /dev/null +++ b/test/e_totalchanges.test @@ -0,0 +1,213 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_totalchanges + +# Like [do_execsql_test], except it appends the value returned by +# [db total_changes] to the result of executing the SQL script. +# +proc do_tc_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db total_changes\]" $res + ] +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX t1_b ON t1(b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX t2_y ON t2(y); +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-65438-26258 This function returns the total number of +# rows inserted, modified or deleted by all INSERT, UPDATE or DELETE +# statements completed since the database connection was opened, +# including those executed as part of trigger programs. +# +# 1.1.*: different types of I/U/D statements, +# 1.2.*: trigger programs. +# +do_tc_test 1.1.1 { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + UPDATE t1 SET a = a+1; + DELETE FROM t1; +} {6} +do_tc_test 1.1.2 { + DELETE FROM t1 +} {6} + +do_tc_test 1.1.3 { + WITH data(a,b) AS ( + SELECT 0, 0 UNION ALL SELECT a+1, b+1 FROM data WHERE a<99 + ) + INSERT INTO t1 SELECT * FROM data; +} {106} + +do_tc_test 1.1.4 { + INSERT INTO t2 SELECT * FROM t1 WHERE a<50; + UPDATE t2 SET y=y+1; +} {206} + +do_tc_test 1.1.5 { + DELETE FROM t2 WHERE y<=25 +} {231} + +do_execsql_test 1.2.1 { + DELETE FROM t1; + DELETE FROM t2; +} +sqlite3 db test.db ; # To reset total_changes +do_tc_test 1.2.2 { + CREATE TABLE log(detail); + CREATE TRIGGER t1_after_insert AFTER INSERT ON t1 BEGIN + INSERT INTO log VALUES('inserted into t1'); + END; + + CREATE TRIGGER t1_before_delete BEFORE DELETE ON t1 BEGIN + INSERT INTO log VALUES('deleting from t1'); + INSERT INTO log VALUES('here we go!'); + END; + + CREATE TRIGGER t1_after_update AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES('update'); + DELETE FROM log; + END; + + INSERT INTO t1 VALUES('a', 'b'); -- 1 + 1 + UPDATE t1 SET b='c'; -- 1 + 1 + 2 + DELETE FROM t1; -- 1 + 1 + 1 +} {9} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-61766-15253 Executing any other type of SQL statement +# does not affect the value returned by sqlite3_total_changes(). +do_tc_test 2.1 { + INSERT INTO t1 VALUES(1, 2), (3, 4); + INSERT INTO t2 VALUES(1, 2), (3, 4); +} {15} +do_tc_test 2.2 { + SELECT count(*) FROM t1; +} {2 15} +do_tc_test 2.3 { + CREATE TABLE t4(a, b); + ALTER TABLE t4 ADD COLUMN c; + CREATE INDEX i4 ON t4(c); + ALTER TABLE t4 RENAME TO t5; + ANALYZE; + BEGIN; + DROP TABLE t2; + ROLLBACK; + VACUUM; +} {15} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-36043-10590 Changes made as part of foreign key +# actions are included in the count, but those made as part of REPLACE +# constraint resolution are not. +# +# 3.1.*: foreign key actions +# 3.2.*: REPLACE constraints. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.1 { + CREATE TABLE p1(c PRIMARY KEY, d); + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.2 { DELETE FROM p1 WHERE c=1; } {9} +do_tc_test 3.1.3 { DELETE FROM p1 WHERE c=2; } {11} +do_tc_test 3.1.4 { DELETE FROM p1 WHERE c=3; } {13} +do_tc_test 3.1.5 { DELETE FROM p1 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.6 { + DROP TABLE c1; + DROP TABLE c2; + DROP TABLE c3; + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.7 { UPDATE p1 SET c=c+4 WHERE c=1; } {9} +do_tc_test 3.1.8 { UPDATE p1 SET c=c+4 WHERE c=2; } {11} +do_tc_test 3.1.9 { UPDATE p1 SET c=c+4 WHERE c=3; } {13} +do_tc_test 3.1.10 { UPDATE p1 SET c=c+4 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.2.1 { + CREATE TABLE t3(a UNIQUE, b UNIQUE); + INSERT INTO t3 VALUES('one', 'one'); + INSERT INTO t3 VALUES('two', 'two'); + INSERT OR REPLACE INTO t3 VALUES('one', 'two'); +} {3} + +do_tc_test 3.2.2 { + INSERT INTO t3 VALUES('three', 'one'); + UPDATE OR REPLACE t3 SET b='two' WHERE b='one'; + SELECT * FROM t3; +} {three two 5} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-54872-08741 Changes to a view that are intercepted by +# INSTEAD OF triggers are not counted. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 4.1 { + CREATE TABLE t6(x); + CREATE VIEW v1 AS SELECT * FROM t6; + CREATE TRIGGER v1_tr1 INSTEAD OF INSERT ON v1 BEGIN + SELECT 'no-op'; + END; + + INSERT INTO v1 VALUES('a'); + INSERT INTO v1 VALUES('b'); +} {0} +do_tc_test 4.2 { + CREATE TRIGGER v1_tr2 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO t6 VALUES(new.x); + END; + + INSERT INTO v1 VALUES('c'); + INSERT INTO v1 VALUES('d'); +} {2} + + +finish_test From 22e8d833f06835a9ea2cdf7e7d2dd55f79102ce0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 00:58:38 +0000 Subject: [PATCH 42/51] In the OP_Column opcode, when extracting a field that is past the end of a short record (because the row was originally inserted prior to ALTER TABLE ADD COLUMN) then make sure the output register is fully NULL and does not contain leftover flags (such as MEM_Ephem) from its previous use. Fix for ticket [43107840f1c02]. FossilOrigin-Name: 24780f8ddc1683fc62180e6961dc6bfe1168f4df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/update.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbbdd001c6..209baf8beb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_totalchanges.test,\scontaining\stests\sof\sthe\ssqlite3_total_changes()\sinterface. -D 2014-10-28T20:49:59.213 +C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02]. +D 2014-10-29T00:58:38.898 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 1b7e8ccaca2a23ae2804568f34b7c645adfd332d +F src/vdbe.c 49e659bc165e99b28492004b440e22146dc898ab F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 @@ -1067,7 +1067,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 -F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb +F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96e9917c350dfe2069b87860bbb961424ff1105a -R ff91cb59544b367ee28dfb499ecb1ca4 -U dan -Z ba0579c9ba051395a01d0b2f40d05365 +P f84af4adcc34d7a4c72027bf5b038a1a45a4c307 +R c2870e402bb1970025fb678b8c13d710 +U drh +Z 3a55744e14b55dcc32cb71f070af972c diff --git a/manifest.uuid b/manifest.uuid index 6aa58547c5..ea9bca8740 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f84af4adcc34d7a4c72027bf5b038a1a45a4c307 \ No newline at end of file +24780f8ddc1683fc62180e6961dc6bfe1168f4df \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index d256c6b770..c17bfdfe14 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2441,7 +2441,7 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } diff --git a/test/update.test b/test/update.test index e67b0efddc..d7baf6e702 100644 --- a/test/update.test +++ b/test/update.test @@ -604,5 +604,19 @@ do_test update-14.4 { } ;# ifcapable {trigger} +# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 +# An assertion fault on UPDATE +# +do_execsql_test update-15.1 { + CREATE TABLE t15(a INTEGER PRIMARY KEY, b); + INSERT INTO t15(a,b) VALUES(10,'abc'),(20,'def'),(30,'ghi'); + ALTER TABLE t15 ADD COLUMN c; + CREATE INDEX t15c ON t15(c); + INSERT INTO t15(a,b) + VALUES(5,'zyx'),(15,'wvu'),(25,'tsr'),(35,'qpo'); + UPDATE t15 SET c=printf("y%d",a) WHERE c IS NULL; + SELECT a,b,c,'|' FROM t15 ORDER BY a; +} {5 zyx y5 | 10 abc y10 | 15 wvu y15 | 20 def y20 | 25 tsr y25 | 30 ghi y30 | 35 qpo y35 |} + finish_test From af8f513f9d9414b8aae11f9470e4a8789e7cf74e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 18:20:18 +0000 Subject: [PATCH 43/51] Fix the %c format character in sqlite3VXPrintf() so that it correctly handles precisions larger than 70. FossilOrigin-Name: 08a27440f19b7fc884464832e6105af1bf008172 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 33 +++++++++++++++++++-------------- src/sqliteInt.h | 2 +- test/printf2.test | 20 ++++++++++++++++++++ 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 209baf8beb..c8c7eafdd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02]. -D 2014-10-29T00:58:38.898 +C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\nhandles\sprecisions\slarger\sthan\s70. +D 2014-10-29T18:20:18.932 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 +F src/printf.c 10a2493593c8e4a538915cd3674bd7a67f70c488 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 +F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -772,7 +772,7 @@ F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 -F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 +F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f84af4adcc34d7a4c72027bf5b038a1a45a4c307 -R c2870e402bb1970025fb678b8c13d710 +P 24780f8ddc1683fc62180e6961dc6bfe1168f4df +R c12283d048765bdbe52bad70443f70c1 U drh -Z 3a55744e14b55dcc32cb71f070af972c +Z 8aeb3112f50d033e0580c4400e41dd66 diff --git a/manifest.uuid b/manifest.uuid index ea9bca8740..bd48a08b43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24780f8ddc1683fc62180e6961dc6bfe1168f4df \ No newline at end of file +08a27440f19b7fc884464832e6105af1bf008172 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index f000da7fcc..85d237f0e7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -212,7 +212,7 @@ void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -336,7 +336,6 @@ void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -627,13 +626,16 @@ void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: @@ -734,11 +736,14 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3_free(zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -791,11 +796,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -void sqlite3AppendSpace(StrAccum *p, int N){ +void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5114ccccf4..f4785411d5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3543,7 +3543,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); -void sqlite3AppendSpace(StrAccum*,int); +void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); diff --git a/test/printf2.test b/test/printf2.test index 4cb1783bfb..21deeb779d 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -95,5 +95,25 @@ do_execsql_test printf2-2.3 { SELECT printf('%s=(%d/%g/%s)',a) FROM t1 ORDER BY a; } {-1=(0/0/) 1=(0/0/) 1.5=(0/0/) abc=(0/0/)} +# The precision of the %c conversion causes the character to repeat. +# +do_execsql_test printf2-3.1 { + SELECT printf('|%110.100c|','*'); +} {{| ****************************************************************************************************|}} +do_execsql_test printf2-3.2 { + SELECT printf('|%-110.100c|','*'); +} {{|**************************************************************************************************** |}} +do_execsql_test printf2-3.3 { + SELECT printf('|%9.8c|%-9.8c|','*','*'); +} {{| ********|******** |}} +do_execsql_test printf2-3.4 { + SELECT printf('|%8.8c|%-8.8c|','*','*'); +} {|********|********|} +do_execsql_test printf2-3.5 { + SELECT printf('|%7.8c|%-7.8c|','*','*'); +} {|********|********|} + + + finish_test From ba0f99941848816581ac12bd1d5f6d03216f5338 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 Oct 2014 20:48:44 +0000 Subject: [PATCH 44/51] Tweaks to comments in btree.c. Minor code changes to enhance testability. FossilOrigin-Name: c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c8c7eafdd4..c8c02f1b16 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\nhandles\sprecisions\slarger\sthan\s70. -D 2014-10-29T18:20:18.932 +C Tweaks\sto\scomments\sin\sbtree.c.\s\sMinor\scode\schanges\sto\senhance\stestability. +D 2014-10-30T20:48:44.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 5189881ca403938c5ceddde496b984fef9f40c5a +F src/btree.c 8d955d8ef15dd724ea5ef1cb65c17151beaff1e0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24780f8ddc1683fc62180e6961dc6bfe1168f4df -R c12283d048765bdbe52bad70443f70c1 +P 08a27440f19b7fc884464832e6105af1bf008172 +R 984c5e022c7b2ce03eb5bed97ff43b89 U drh -Z 8aeb3112f50d033e0580c4400e41dd66 +Z 333c45d7dfd79a88b32c00dfbc4028a5 diff --git a/manifest.uuid b/manifest.uuid index bd48a08b43..8964ddfef6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -08a27440f19b7fc884464832e6105af1bf008172 \ No newline at end of file +c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 22b168d9e4..e56cdf81ed 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1231,10 +1231,8 @@ static int defragmentPage(MemPage *pPage){ ** ** If no suitable space can be found on the free-list, return NULL. ** -** This function may detect corruption within pPg. If it does and argument -** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected and pRc is NULL, NULL is returned and the -** corruption goes unreported. +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** ** If a slot of at least nByte bytes is found but cannot be used because ** there are already at least 60 fragmented bytes on the page, return NULL. @@ -1250,7 +1248,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ int size; /* Size of the free slot */ if( pc>usableSize-4 || pc usableSize ){ - if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + *pRc = SQLITE_CORRUPT_BKPT; return 0; }else{ /* The slot remains on the free-list. Reduce its size to account @@ -6076,8 +6074,9 @@ static int pageInsertArray( assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; i Date: Thu, 30 Oct 2014 23:14:56 +0000 Subject: [PATCH 45/51] Improvements to the wording of some comments. Reinstate an assert() that is only true for non-corrupt database files by adding an "|| CORRUPT_DB" term. FossilOrigin-Name: 67adb44838f98805f86aecca634d9a3b07370b9e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 27 ++++++++++++--------------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index c8c02f1b16..98a273f2e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\scomments\sin\sbtree.c.\s\sMinor\scode\schanges\sto\senhance\stestability. -D 2014-10-30T20:48:44.305 +C Improvements\sto\sthe\swording\sof\ssome\scomments.\s\sReinstate\san\sassert()\sthat\nis\sonly\strue\sfor\snon-corrupt\sdatabase\sfiles\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. +D 2014-10-30T23:14:56.756 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 8d955d8ef15dd724ea5ef1cb65c17151beaff1e0 +F src/btree.c 8f7ea96935c3b1db94439204965a6ec1392f7977 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 08a27440f19b7fc884464832e6105af1bf008172 -R 984c5e022c7b2ce03eb5bed97ff43b89 +P c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 +R 230a6673a34b4786200f7ae54e553a63 U drh -Z 333c45d7dfd79a88b32c00dfbc4028a5 +Z 52f8e66a5796212d2c4dbab1cb7cef14 diff --git a/manifest.uuid b/manifest.uuid index 8964ddfef6..fd375ba77a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 \ No newline at end of file +67adb44838f98805f86aecca634d9a3b07370b9e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e56cdf81ed..2c3cbb4adc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6526,7 +6526,7 @@ static int balance_nonroot( u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ @@ -6765,9 +6765,10 @@ static int balance_nonroot( /* ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. ** ** This adjustment is more than an optimization. The packing above might ** be so out of balance as to be illegal. For example, the right-most @@ -6796,18 +6797,14 @@ static int balance_nonroot( szNew[i-1] = szLeft; } - /* Either we found one or more cells (cntnew[0])>0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - ** - ** UPDATE: The assert() below is not necessarily true if the database - ** file is corrupt. The corruption will be detected and reported later - ** in this procedure so there is no need to act upon it now. + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. */ -#if 0 - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); -#endif - + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, From 00fe08af82c5e66a69abfbf687d9d99d46d8ad47 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 00:05:23 +0000 Subject: [PATCH 46/51] In the balance_nonroot() routine, protect the values in aPgno[] array from change during the page sort, so that aPgno[] can be used to avoid unnecessary pointer-map updates for auto_vacuum databases. FossilOrigin-Name: 69c3924fe834a78d4a8d86833626bf5f68e33a3a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 98a273f2e0..b3f68e9802 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\swording\sof\ssome\scomments.\s\sReinstate\san\sassert()\sthat\nis\sonly\strue\sfor\snon-corrupt\sdatabase\sfiles\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. -D 2014-10-30T23:14:56.756 +C In\sthe\sbalance_nonroot()\sroutine,\sprotect\sthe\svalues\sin\saPgno[]\sarray\sfrom\nchange\sduring\sthe\spage\ssort,\sso\sthat\saPgno[]\scan\sbe\sused\sto\savoid\sunnecessary\npointer-map\supdates\sfor\sauto_vacuum\sdatabases. +D 2014-10-31T00:05:23.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 8f7ea96935c3b1db94439204965a6ec1392f7977 +F src/btree.c 88c87803b334807da3150245987139baacea4e6e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 -R 230a6673a34b4786200f7ae54e553a63 +P 67adb44838f98805f86aecca634d9a3b07370b9e +R 882446b4b0e231e5d89ec5bd9f22a940 U drh -Z 52f8e66a5796212d2c4dbab1cb7cef14 +Z 021277b745352cb31d525498ae57f1ac diff --git a/manifest.uuid b/manifest.uuid index fd375ba77a..03860ab39c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67adb44838f98805f86aecca634d9a3b07370b9e \ No newline at end of file +69c3924fe834a78d4a8d86833626bf5f68e33a3a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2c3cbb4adc..05dbd8cec7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6533,6 +6533,7 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); @@ -6859,7 +6860,7 @@ static int balance_nonroot( ** for large insertions and deletions. */ for(i=0; ipgno; + aPgOrder[i] = aPgno[i] = apNew[i]->pgno; aPgFlags[i] = apNew[i]->pDbPage->flags; for(j=0; ji ){ sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); From 768f29002e24301bb601e5eefcf41b1a0904b78f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 02:51:41 +0000 Subject: [PATCH 47/51] Remove an unnecessary branch from balance_nonroot(). FossilOrigin-Name: 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 23 ++++++++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index b3f68e9802..2496dc7f00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sbalance_nonroot()\sroutine,\sprotect\sthe\svalues\sin\saPgno[]\sarray\sfrom\nchange\sduring\sthe\spage\ssort,\sso\sthat\saPgno[]\scan\sbe\sused\sto\savoid\sunnecessary\npointer-map\supdates\sfor\sauto_vacuum\sdatabases. -D 2014-10-31T00:05:23.983 +C Remove\san\sunnecessary\sbranch\sfrom\sbalance_nonroot(). +D 2014-10-31T02:51:41.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 88c87803b334807da3150245987139baacea4e6e +F src/btree.c 7031b8cec28b4ba853090da021c6c456952f1f92 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67adb44838f98805f86aecca634d9a3b07370b9e -R 882446b4b0e231e5d89ec5bd9f22a940 +P 69c3924fe834a78d4a8d86833626bf5f68e33a3a +R 85b30d88694d9feacf26ae3f80208908 U drh -Z 021277b745352cb31d525498ae57f1ac +Z 7bc5c33b6cb165c738435440761728db diff --git a/manifest.uuid b/manifest.uuid index 03860ab39c..d59874b337 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69c3924fe834a78d4a8d86833626bf5f68e33a3a \ No newline at end of file +9fc7c88e3f5221883aa6eafbf8af3be94db0c299 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 05dbd8cec7..f21fce2a51 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7078,19 +7078,20 @@ static int balance_nonroot( ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ assert( nNew==1 ); rc = defragmentPage(apNew[0]); - if( rc==SQLITE_OK ){ - assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); - } + testcase( rc!=SQLITE_OK ); + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + || rc!=SQLITE_OK + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 9c0153457af66ca84768c08472f7424f7e6711d0 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 10:31:59 +0000 Subject: [PATCH 48/51] Add the "varint.c" utility program in the tool directory. FossilOrigin-Name: ea5d56be5fe14934e4dbe9c17d46b058f487a231 --- manifest | 11 +++-- manifest.uuid | 2 +- tool/varint.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 tool/varint.c diff --git a/manifest b/manifest index 2496dc7f00..2c4f7bf41f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sbranch\sfrom\sbalance_nonroot(). -D 2014-10-31T02:51:41.215 +C Add\sthe\s"varint.c"\sutility\sprogram\sin\sthe\stool\sdirectory. +D 2014-10-31T10:31:59.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,12 +1204,13 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 +F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69c3924fe834a78d4a8d86833626bf5f68e33a3a -R 85b30d88694d9feacf26ae3f80208908 +P 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 +R 932c8189456e662bc00860a65a53ebcf U drh -Z 7bc5c33b6cb165c738435440761728db +Z 653e5ed37198e17f610045be3bfdae23 diff --git a/manifest.uuid b/manifest.uuid index d59874b337..337c7347f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fc7c88e3f5221883aa6eafbf8af3be94db0c299 \ No newline at end of file +ea5d56be5fe14934e4dbe9c17d46b058f487a231 \ No newline at end of file diff --git a/tool/varint.c b/tool/varint.c new file mode 100644 index 0000000000..f4a51118b4 --- /dev/null +++ b/tool/varint.c @@ -0,0 +1,123 @@ +/* +** A utility program to translate SQLite varints into decimal and decimal +** integers into varints. +*/ +#include +#include +#include + +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 i64; + typedef unsigned __int64 u64; +#else + typedef long long int i64; + typedef unsigned long long int u64; +#endif + +static int hexValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +static char toHex(unsigned char c){ + return "0123456789abcdef"[c&0xf]; +} + +static int putVarint(unsigned char *p, u64 v){ + int i, j, n; + unsigned char buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + + +int main(int argc, char **argv){ + int i; + u64 x; + u64 uX = 0; + i64 iX; + int n; + unsigned char zHex[20]; + + if( argc==1 ){ + fprintf(stderr, + "Usage:\n" + " %s HH HH HH ... Convert varint to decimal\n" + " %s DDDDD Convert decimal to varint\n" + " Add '+' or '-' before DDDDD to disambiguate.\n", + argv[0], argv[0]); + exit(1); + } + if( argc>2 + || (strlen(argv[1])==2 && hexValue(argv[1][0])>=0 && hexValue(argv[1][1])>=0) + ){ + /* Hex to decimal */ + for(i=1; i'9' ){ + fprintf(stderr, "Not a decimal number: %s", argv[1]); + exit(1); + } + uX = uX*10 + z[0] - '0'; + z++; + } + if( sign<0 ){ + memcpy(&iX, &uX, 8); + iX = -iX; + memcpy(&uX, &iX, 8); + } + } + n = putVarint(zHex, uX); + printf("%lld =", (i64)uX); + for(i=0; i>4), toHex(zHex[i]&0x0f)); + } + printf("\n"); + return 0; +} From bec021b9fca398564dd2eccabe10d8af8fa6fc41 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 12:22:00 +0000 Subject: [PATCH 49/51] Simplify the math slightly, and reduce by one the number of loop iterations, for the loop in balance_nonroot() that moves cells between pages. FossilOrigin-Name: 2e838db82e533598b3cb00011c04fc0d5a896895 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2c4f7bf41f..e6108baa44 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"varint.c"\sutility\sprogram\sin\sthe\stool\sdirectory. -D 2014-10-31T10:31:59.052 +C Simplify\sthe\smath\sslightly,\sand\sreduce\sby\sone\sthe\snumber\sof\sloop\siterations,\nfor\sthe\sloop\sin\sbalance_nonroot()\sthat\smoves\scells\sbetween\spages. +D 2014-10-31T12:22:00.360 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7031b8cec28b4ba853090da021c6c456952f1f92 +F src/btree.c 0f294d5e1eca7a912037d1992c904e28959a5aa5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 -R 932c8189456e662bc00860a65a53ebcf +P ea5d56be5fe14934e4dbe9c17d46b058f487a231 +R 5dd41c1bf29a3c6eca71e97060df99dc U drh -Z 653e5ed37198e17f610045be3bfdae23 +Z 53accc48044e0d3a03ff14711aee89d8 diff --git a/manifest.uuid b/manifest.uuid index 337c7347f8..000bb51e9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea5d56be5fe14934e4dbe9c17d46b058f487a231 \ No newline at end of file +2e838db82e533598b3cb00011c04fc0d5a896895 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f21fce2a51..4176d2c8bd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7036,8 +7036,10 @@ static int balance_nonroot( ** ** If neither of the above apply, the page is safe to update. */ - for(i=0; i=nNew ? i-nNew : nNew-1-i); + for(i=1-nNew; i=0 && iPg=cntNew[iPg-1] || abDone[iPg-1]) && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) From d836d42383d93c549826510747b0b6a1bb55c55b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 14:26:36 +0000 Subject: [PATCH 50/51] Simplify the logic in the cell redistribution loop of balance_nonroot(). Enhance and clarify comments and add assert() statements for additional verification of correctness. FossilOrigin-Name: a07078b60007e88adea67bec5f0caf91f707ad78 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 39 ++++++++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index e6108baa44..5aa7ed2f46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\smath\sslightly,\sand\sreduce\sby\sone\sthe\snumber\sof\sloop\siterations,\nfor\sthe\sloop\sin\sbalance_nonroot()\sthat\smoves\scells\sbetween\spages. -D 2014-10-31T12:22:00.360 +C Simplify\sthe\slogic\sin\sthe\scell\sredistribution\sloop\sof\sbalance_nonroot().\nEnhance\sand\sclarify\scomments\sand\sadd\sassert()\sstatements\sfor\sadditional\nverification\sof\scorrectness. +D 2014-10-31T14:26:36.417 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 0f294d5e1eca7a912037d1992c904e28959a5aa5 +F src/btree.c 61d96c2edacc5267fae6e477c24c774cd540d7f0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ea5d56be5fe14934e4dbe9c17d46b058f487a231 -R 5dd41c1bf29a3c6eca71e97060df99dc +P 2e838db82e533598b3cb00011c04fc0d5a896895 +R 6da135d51be6a673f0389acc80c0dfb3 U drh -Z 53accc48044e0d3a03ff14711aee89d8 +Z 452523febd435385cb08d146713a8a8f diff --git a/manifest.uuid b/manifest.uuid index 000bb51e9a..4ff01299eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e838db82e533598b3cb00011c04fc0d5a896895 \ No newline at end of file +a07078b60007e88adea67bec5f0caf91f707ad78 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4176d2c8bd..9300a6a54f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7026,28 +7026,43 @@ static int balance_nonroot( ** is important, as this code needs to avoid disrupting any page from which ** cells may still to be read. In practice, this means: ** - ** 1) If cells are to be removed from the start of the page and shifted - ** to the left-hand sibling, it is not safe to update the page until - ** the left-hand sibling (apNew[i-1]) has already been updated. + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. ** - ** 2) If cells are to be removed from the end of the page and shifted - ** to the right-hand sibling, it is not safe to update the page until - ** the right-hand sibling (apNew[i+1]) has already been updated. + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. ** ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. */ for(i=1-nNew; i=0 && iPg=cntNew[iPg-1] || abDone[iPg-1]) - && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) + if( abDone[iPg] ) continue; /* Skip pages already processed */ + if( i>=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ ){ int iNew; int iOld; int nNewCell; + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + if( iPg==0 ){ iNew = iOld = 0; nNewCell = cntNew[0]; @@ -7058,12 +7073,14 @@ static int balance_nonroot( } editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); - abDone[iPg] = 1; + abDone[iPg]++; apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } } + + /* All pages have been processed exactly once */ assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); assert( nOld>0 ); From 0fb5daed34ba6f16d761b8ad02b944ec59cf5c03 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 14:46:51 +0000 Subject: [PATCH 51/51] Change the command-line shell man-page to use the ".tr" troff directive instead of ".cc" for escaping the initial "." characters in the ".help" output. FossilOrigin-Name: 67f0d469da28c023200239a1f3d0c6cef9ef0e45 --- manifest | 12 ++++---- manifest.uuid | 2 +- sqlite3.1 | 83 ++++++++++++++++++++++++++++----------------------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 5aa7ed2f46..ce0614fbee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\slogic\sin\sthe\scell\sredistribution\sloop\sof\sbalance_nonroot().\nEnhance\sand\sclarify\scomments\sand\sadd\sassert()\sstatements\sfor\sadditional\nverification\sof\scorrectness. -D 2014-10-31T14:26:36.417 +C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. +D 2014-10-31T14:46:51.896 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 F mptest/multiwrite01.test 499ad0310da8dff8e8f98d2e272fc2a8aa741b2e F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b -F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 +F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2e838db82e533598b3cb00011c04fc0d5a896895 -R 6da135d51be6a673f0389acc80c0dfb3 +P a07078b60007e88adea67bec5f0caf91f707ad78 +R f904ce78baced6aeb6bcb91c631714db U drh -Z 452523febd435385cb08d146713a8a8f +Z c463256d5d45e3bce4a212ca4dd9f3e4 diff --git a/manifest.uuid b/manifest.uuid index 4ff01299eb..4735237141 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a07078b60007e88adea67bec5f0caf91f707ad78 \ No newline at end of file +67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file diff --git a/sqlite3.1 b/sqlite3.1 index 02df7f44c7..80353b0eec 100644 --- a/sqlite3.1 +++ b/sqlite3.1 @@ -2,7 +2,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH SQLITE3 1 "Mon Jan 31 11:14:00 2014" +.TH SQLITE3 1 "Fri Oct 31 10:41:31 EDT 2014" .\" Please adjust this date whenever revising the manpage. .\" .\" Some roff macros, for reference: @@ -49,7 +49,7 @@ a table named "memos" and insert a couple of records into that table: $ .B sqlite3 mydata.db .br -SQLite version 3.8.3 +SQLite version 3.8.8 .br Enter ".help" for instructions .br @@ -107,26 +107,29 @@ the '.help' command. For example: sqlite> .B .help .nf -.cc | -.backup ?DB? FILE Backup DB (default "main") to FILE -.bail ON|OFF Stop after hitting an error. Default OFF -.databases List names and files of attached databases -.dump ?TABLE? ... Dump the database in an SQL text format +.tr %. +%backup ?DB? FILE Backup DB (default "main") to FILE +%bail on|off Stop after hitting an error. Default OFF +%clone NEWDB Clone data into NEWDB from the existing database +%databases List names and files of attached databases +%dump ?TABLE? ... Dump the database in an SQL text format If TABLE specified, only dump tables matching LIKE pattern TABLE. -.echo ON|OFF Turn command echo on or off -.exit Exit this program -.explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off. +%echo on|off Turn command echo on or off +%eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN +%exit Exit this program +%explain ?on|off? Turn output mode suitable for EXPLAIN on or off. With no args, it turns EXPLAIN on. -.header(s) ON|OFF Turn display of headers on or off -.help Show this message -.import FILE TABLE Import data from FILE into TABLE -.indices ?TABLE? Show names of all indices +%fullschema Show schema and the content of sqlite_stat tables +%headers on|off Turn display of headers on or off +%help Show this message +%import FILE TABLE Import data from FILE into TABLE +%indices ?TABLE? Show names of all indices If TABLE specified, only show indices for tables matching LIKE pattern TABLE. -.load FILE ?ENTRY? Load an extension library -.log FILE|off Turn logging on or off. FILE can be stderr/stdout -.mode MODE ?TABLE? Set output mode where MODE is one of: +%load FILE ?ENTRY? Load an extension library +%log FILE|off Turn logging on or off. FILE can be stderr/stdout +%mode MODE ?TABLE? Set output mode where MODE is one of: csv Comma-separated values column Left-aligned columns. (See .width) html HTML code @@ -135,31 +138,35 @@ sqlite> list Values delimited by .separator string tabs Tab-separated values tcl TCL list elements -.nullvalue STRING Use STRING in place of NULL values -.open ?FILENAME? Close existing database and reopen FILENAME -.output FILENAME Send output to FILENAME -.output stdout Send output to the screen -.print STRING... Print literal STRING -.prompt MAIN CONTINUE Replace the standard prompts -.quit Exit this program -.read FILENAME Execute SQL in FILENAME -.restore ?DB? FILE Restore content of DB (default "main") from FILE -.schema ?TABLE? Show the CREATE statements +%nullvalue STRING Use STRING in place of NULL values +%once FILENAME Output for the next SQL command only to FILENAME +%open ?FILENAME? Close existing database and reopen FILENAME +%output ?FILENAME? Send output to FILENAME or stdout +%print STRING... Print literal STRING +%prompt MAIN CONTINUE Replace the standard prompts +%quit Exit this program +%read FILENAME Execute SQL in FILENAME +%restore ?DB? FILE Restore content of DB (default "main") from FILE +%save FILE Write in-memory database into FILE +%schema ?TABLE? Show the CREATE statements If TABLE specified, only show tables matching LIKE pattern TABLE. -.separator STRING Change separator used by output mode and .import -.show Show the current values for various settings -.stats ON|OFF Turn stats on or off -.tables ?TABLE? List names of tables +%separator STRING ?NL? Change separator used by output mode and .import + NL is the end-of-line mark for CSV +%shell CMD ARGS... Run CMD ARGS... in a system shell +%show Show the current values for various settings +%stats on|off Turn stats on or off +%system CMD ARGS... Run CMD ARGS... in a system shell +%tables ?TABLE? List names of tables If TABLE specified, only list tables matching LIKE pattern TABLE. -.timeout MS Try opening locked tables for MS milliseconds -.trace FILE|off Output each SQL statement as it is run -.vfsname ?AUX? Print the name of the VFS stack -.width NUM1 NUM2 ... Set column widths for "column" mode -.timer ON|OFF Turn the CPU timer measurement on or off +%timeout MS Try opening locked tables for MS milliseconds +%timer on|off Turn SQL timer on or off +%trace FILE|off Output each SQL statement as it is run +%vfsname ?AUX? Print the name of the VFS stack +%width NUM1 NUM2 ... Set column widths for "column" mode + Negative values right-justify sqlite> -|cc . .sp .fi .SH OPTIONS @@ -269,7 +276,7 @@ o If the -init option is present, the specified file is processed. o All other command line options are processed. .SH SEE ALSO -http://www.sqlite.org/ +http://www.sqlite.org/cli.html .br The sqlite3-doc package. .SH AUTHOR