From 3460d19c852885545bb86d92d5c3dd8f12b84990 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Sat, 27 Dec 2008 15:23:13 +0000 Subject: [PATCH] Fix a problem with savepoint and incremental-vacuum. (CVS 6066) FossilOrigin-Name: 08352f9ea9d2a1759320efc46e418079000855cb --- manifest | 22 +++--- manifest.uuid | 2 +- src/btree.c | 109 ++++++++++------------------- src/btreeInt.h | 3 +- src/pager.c | 162 ++++++++++++++++++++++++++++++------------- src/pager.h | 7 +- test/incrvacuum.test | 44 +++++++++++- test/savepoint.test | 95 ++++++++++++++++++++++++- 8 files changed, 306 insertions(+), 138 deletions(-) diff --git a/manifest b/manifest index 75e16952e7..82ed58fb8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reset\sthe\scolumn\scache\sbefore\scoding\seach\sstep\sof\sa\strigger\sprogram.\sCandidate\sfix\sfor\s#3554.\s(CVS\s6065) -D 2008-12-26T07:56:39 +C Fix\sa\sproblem\swith\ssavepoint\sand\sincremental-vacuum.\s(CVS\s6066) +D 2008-12-27T15:23:13 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 77635d0909c2067cee03889a1e04ce910d8fb809 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -103,9 +103,9 @@ F src/attach.c 1c35f95da3c62d19de75b44cfefd12c81c1791b3 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627 F src/bitvec.c 4300d311b17fb3c1476623fd895a8feac02a0b08 F src/btmutex.c 63c5cc4ad5715690767ffcb741e185d7bc35ec1a -F src/btree.c f695109b39fc104bd2f904b1a2d483d44faf9c86 +F src/btree.c 581fdccd7b6539a8e37e843f9b45e0557b3b272f F src/btree.h 4f141cf748d2ee7c6d7fc175f64f87a45cd44113 -F src/btreeInt.h 7ef2c872371d7508657f8d7a4efe651c741d6ee6 +F src/btreeInt.h 8fea5cd7021cb8848fc2183a3e909469659daa0a F src/build.c 92335a6c6a7c119580be605c5dd1439602d6cf5d F src/callback.c bee8949d619b1b7b1e4dfac8a19c5116ae1dd12a F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c @@ -142,8 +142,8 @@ F src/os_common.h 24525d8b7bce66c374dfc1810a6c9043f3359b60 F src/os_os2.c bed77dc26e3a95ce4a204936b9a1ca6fe612fcc5 F src/os_unix.c e6eacc7ec735ded605fefcbaf250058baa8feb12 F src/os_win.c 496e3ceb499aedc63622a89ef76f7af2dd902709 -F src/pager.c 67af495bcd0723a1574dcdefebeac8edc3fee492 -F src/pager.h 7191294438881eb4d13eedade97891e8dc993905 +F src/pager.c ec486337fae32dfcaac41f4471c70a0385fe8487 +F src/pager.h 0579740d4c18826b46124c82330467b41f407eb1 F src/parse.y 4d0e33a702dc3ea7b69d8ae1914b3fbd32e46057 F src/pcache.c 16dc8da6e6ba6250f8dfd9ee46036db1cbceedc6 F src/pcache.h f20c3e82dd6da622c3fe296170cb1801f9a2d75a @@ -385,7 +385,7 @@ F test/in4.test 9bfd9226a82ac832046abc93acecf42627ebb45a F test/incrblob.test 4b9437bbb38724343dadbbcca6356bc2a9b435d1 F test/incrblob2.test 5cca1c3cb29064c504b3b0cc3e2cd43e8053cfdf F test/incrblob_err.test c577c91d4ed9e8336cdb188b15d6ee2a6fe9604e -F test/incrvacuum.test 9a6346c56ffa141024054ae7ba6c8655edf2d137 +F test/incrvacuum.test 6ef5877f26d1e7bc9bc137ea8f8f069a260ff9c6 F test/incrvacuum2.test 46ef65f377e3937cfd1ba66e818309dab46f590d F test/incrvacuum_ioerr.test 57d2f5777ab13fa03b87b262a4ea1bad5cfc0291 F test/index.test cbf301cdb2da43e4eac636c3400c2439af1834ad @@ -492,7 +492,7 @@ F test/rollback.test 1f70ab4301d8d105d41438a436cad1fc8897f5e5 F test/rowid.test 1c8fc43c60d273e6ea44dfb992db587f3164312c F test/rtree.test b85fd4f0861a40ca366ac195e363be2528dcfadf F test/safety.test b69e2b2dd5d52a3f78e216967086884bbc1a09c6 -F test/savepoint.test 5d43369333ab373c4bde69f090a1659c8d2275de +F test/savepoint.test d00fe3f82773266410799aad0080426a6a936170 F test/savepoint2.test 18f6c75d5c133b93838019df8988b8cdf379d3de F test/savepoint3.test b3c9aa5af3f777ccb8b9e15597c75c93eb5bc369 F test/savepoint4.test fd8850063e3c40565545f5c291e7f79a30591670 @@ -686,7 +686,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e -P c6fd3b8f29927c0fc634f82885f144c78f0105d9 -R c25735f44357d80a804bf8dbd5007586 +P a1b1f6cd7d2c060bd75ce39347e1220b872806ed +R 1a9ec7089b4909b30df347c1e2ee32e6 U danielk1977 -Z e5dd7e980e4e7bfee46bc37a1c56ba2d +Z 75549ad37a08cbb85b58f7a2c8f6829a diff --git a/manifest.uuid b/manifest.uuid index b3333d0017..dd8c9e131b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1b1f6cd7d2c060bd75ce39347e1220b872806ed \ No newline at end of file +08352f9ea9d2a1759320efc46e418079000855cb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0f9b0a52cb..c018d5b145 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.552 2008/12/23 15:58:06 drh Exp $ +** $Id: btree.c,v 1.553 2008/12/27 15:23:13 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. @@ -2280,15 +2280,10 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** number of pages the database file will contain after this ** process is complete. */ -static int incrVacuumStep(BtShared *pBt, Pgno nFin){ - Pgno iLastPg; /* Last page in the database */ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ Pgno nFreeList; /* Number of pages still on the free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); - iLastPg = pBt->nTrunc; - if( iLastPg==0 ){ - iLastPg = pagerPagecount(pBt); - } if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ int rc; @@ -2362,9 +2357,12 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin){ } } - pBt->nTrunc = iLastPg - 1; - while( pBt->nTrunc==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, pBt->nTrunc) ){ - pBt->nTrunc--; + if( nFin==0 ){ + iLastPg--; + while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ + iLastPg--; + } + sqlite3PagerTruncateImage(pBt->pPager, iLastPg); } return SQLITE_OK; } @@ -2388,7 +2386,7 @@ int sqlite3BtreeIncrVacuum(Btree *p){ rc = SQLITE_DONE; }else{ invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, 0); + rc = incrVacuumStep(pBt, 0, sqlite3PagerImageSize(pBt->pPager)); } sqlite3BtreeLeave(p); return rc; @@ -2403,7 +2401,7 @@ int sqlite3BtreeIncrVacuum(Btree *p){ ** i.e. the database has been reorganized so that only the first *pnTrunc ** pages are in use. */ -static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){ +static int autoVacuumCommit(BtShared *pBt){ int rc = SQLITE_OK; Pager *pPager = pBt->pPager; VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); @@ -2412,53 +2410,44 @@ static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){ invalidateAllOverflowCache(pBt); assert(pBt->autoVacuum); if( !pBt->incrVacuum ){ - Pgno nFin = 0; + Pgno nFin; + Pgno nFree; + Pgno nPtrmap; + Pgno iFree; + const int pgsz = pBt->pageSize; + Pgno nOrig = pagerPagecount(pBt); - if( pBt->nTrunc==0 ){ - Pgno nFree; - Pgno nPtrmap; - const int pgsz = pBt->pageSize; - Pgno nOrig = pagerPagecount(pBt); - - if( PTRMAP_ISPAGE(pBt, nOrig) ){ - return SQLITE_CORRUPT_BKPT; - } - if( nOrig==PENDING_BYTE_PAGE(pBt) ){ - nOrig--; - } - nFree = get4byte(&pBt->pPage1->aData[36]); - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5); - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } - while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } + if( PTRMAP_ISPAGE(pBt, nOrig) ){ + return SQLITE_CORRUPT_BKPT; + } + if( nOrig==PENDING_BYTE_PAGE(pBt) ){ + nOrig--; + } + nFree = get4byte(&pBt->pPage1->aData[36]); + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5); + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ + nFin--; } - while( rc==SQLITE_OK ){ - rc = incrVacuumStep(pBt, nFin); + for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ + rc = incrVacuumStep(pBt, nFin, iFree); } - if( rc==SQLITE_DONE ){ - assert(nFin==0 || pBt->nTrunc==0 || nFin<=pBt->nTrunc); + if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = SQLITE_OK; - if( pBt->nTrunc && nFin ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[32], 0); - put4byte(&pBt->pPage1->aData[36], 0); - pBt->nTrunc = nFin; - } + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + sqlite3PagerTruncateImage(pBt->pPager, nFin); } if( rc!=SQLITE_OK ){ sqlite3PagerRollback(pPager); } } - if( rc==SQLITE_OK ){ - *pnTrunc = pBt->nTrunc; - pBt->nTrunc = 0; - } assert( nRef==sqlite3PagerRefcount(pPager) ); return rc; } @@ -2500,7 +2489,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ pBt->db = p->db; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - rc = autoVacuumCommit(pBt, &nTrunc); + rc = autoVacuumCommit(pBt); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); return rc; @@ -2677,10 +2666,6 @@ int sqlite3BtreeRollback(Btree *p){ if( p->inTrans==TRANS_WRITE ){ int rc2; -#ifndef SQLITE_OMIT_AUTOVACUUM - pBt->nTrunc = 0; -#endif - assert( TRANS_WRITE==pBt->inTransaction ); rc2 = sqlite3PagerRollback(pBt->pPager); if( rc2!=SQLITE_OK ){ @@ -4329,16 +4314,6 @@ static int allocateBtreePage( *pPgno = nPage + 1; #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->nTrunc ){ - /* An incr-vacuum has already run within this transaction. So the - ** page to allocate is not from the physical end of the file, but - ** at pBt->nTrunc. - */ - *pPgno = pBt->nTrunc+1; - if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ - (*pPgno)++; - } - } if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){ /* If *pPgno refers to a pointer-map page, allocate two new pages ** at the end of the file instead of one. The first allocated page @@ -4349,9 +4324,6 @@ static int allocateBtreePage( (*pPgno)++; if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ (*pPgno)++; } } - if( pBt->nTrunc ){ - pBt->nTrunc = *pPgno; - } #endif assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); @@ -7059,11 +7031,6 @@ char *sqlite3BtreeIntegrityCheck( sCheck.nErr = 0; sCheck.mallocFailed = 0; *pnErr = 0; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->nTrunc!=0 ){ - sCheck.nPage = pBt->nTrunc; - } -#endif if( sCheck.nPage==0 ){ unlockBtreeIfUnused(pBt); sqlite3BtreeLeave(p); diff --git a/src/btreeInt.h b/src/btreeInt.h index 4e4fefd3ba..17453d2e26 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btreeInt.h,v 1.37 2008/12/10 16:45:51 drh Exp $ +** $Id: btreeInt.h,v 1.38 2008/12/27 15:23:13 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -371,7 +371,6 @@ struct BtShared { #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ - Pgno nTrunc; /* Non-zero if the db will be truncated (incr vacuum) */ #endif u16 pageSize; /* Total number of bytes on a page */ u16 usableSize; /* Number of usable bytes on each page */ diff --git a/src/pager.c b/src/pager.c index 02467c7ca2..ac42d13ff5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.523 2008/12/23 19:15:57 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.524 2008/12/27 15:23:13 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -172,6 +172,18 @@ struct PagerSavepoint { ** next successful rollback is performed on the pager cache. Also, ** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() ** APIs, they may still be used successfully. +** +** Managing the size of the database file in pages is a little complicated. +** The variable Pager.dbSize contains the number of pages that the database +** image currently contains. As the database image grows or shrinks this +** variable is updated. The variable Pager.dbFileSize contains the number +** of pages in the database file. This may be different from Pager.dbSize +** if some pages have been appended to the database image but not yet written +** out from the cache to the actual file on disk. Or if the image has been +** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable +** contains the number of pages in the database image when the current +** transaction was opened. The contents of all three of these variables is +** only guaranteed to be correct if the boolean Pager.dbSizeValid is true. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -196,10 +208,11 @@ struct Pager { u8 dbModified; /* True if there are any changes to the Db */ u8 changeCountDone; /* Set after incrementing the change-counter */ u8 dbSizeValid; /* Set when dbSize is correct */ + Pgno dbSize; /* Number of pages in the database */ + Pgno dbOrigSize; /* dbSize before the current transaction */ + Pgno dbFileSize; /* Number of pages in the database file */ u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ int errCode; /* One of several kinds of errors */ - Pgno dbSize; /* Number of pages in the file */ - Pgno origDbSize; /* dbSize before the current change */ int nRec; /* Number of pages written to the journal */ u32 cksumInit; /* Quasi-random value added to every checksum */ int stmtNRec; /* Number of records in stmt subjournal */ @@ -322,28 +335,30 @@ static const unsigned char aJournalMagic[] = { #define PAGER_MAX_PGNO 2147483647 /* -** Return false if it is necessary to write page *pPg into the sub-journal. -** More accurately, true is returned if either: +** Return true if it is necessary to write page *pPg into the sub-journal. +** A page needs to be written into the sub-journal if there exists one +** or more open savepoints for which: ** -** * No savepoints are open, or -** * The page has been saved to the sub-journal since the most recent -** savepoint was opened. -** -** TODO: There's a bug here. To do with PagerSavepoint.nOrig. Also consider -** the idea that the page may not be required by the outermost savepoint -** but may be required by some earlier savepoint, due to an incremental -** vacuum operation. +** * The page-number is less than or equal to PagerSavepoint.nOrig, and +** * The bit corresponding to the page-number is not set in +** PagerSavepoint.pInSavepoint. */ -static int pageInSavepoint(PgHdr *pPg){ +static int subjRequiresPage(PgHdr *pPg){ + Pgno pgno = pPg->pgno; Pager *pPager = pPg->pPager; - if( pPager->nSavepoint==0 ){ - return 1; + int i; + for(i=0; inSavepoint; i++){ + PagerSavepoint *p = &pPager->aSavepoint[i]; + if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + return 1; + } } - return sqlite3BitvecTest( - pPager->aSavepoint[pPager->nSavepoint-1].pInSavepoint, pPg->pgno - ); + return 0; } +/* +** Return true if the page is already in the journal file. +*/ static int pageInJournal(PgHdr *pPg){ return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno); } @@ -715,7 +730,7 @@ static int writeJournalHdr(Pager *pPager){ sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); /* The initial database size */ - put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbSize); + put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); /* The assumed sector size for this process */ put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); @@ -996,7 +1011,7 @@ static void pager_unlock(Pager *pPager){ releaseAllSavepoint(pPager); pPager->journalOff = 0; pPager->journalStarted = 0; - pPager->origDbSize = 0; + pPager->dbOrigSize = 0; } pPager->state = PAGER_UNLOCK; @@ -1089,7 +1104,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ }else if( pPager->state==PAGER_SYNCED ){ pPager->state = PAGER_EXCLUSIVE; } - pPager->origDbSize = 0; + pPager->dbOrigSize = 0; pPager->setMaster = 0; pPager->needSync = 0; /* lruListSetFirstSynced(pPager); */ @@ -1232,6 +1247,9 @@ static int pager_playback_one_page( ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst); + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } } if( pPg ){ /* No page should ever be explicitly rolled back that is in use, except @@ -1417,6 +1435,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ }else{ rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1); } + if( rc==SQLITE_OK ){ + pPager->dbFileSize = nPage; + } } } if( rc==SQLITE_OK ){ @@ -1654,7 +1675,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ /* Truncate the database back to the size it was before the ** savepoint being reverted was opened. */ - rc = pager_truncate(pPager, pSavepoint?pSavepoint->nOrig:pPager->origDbSize); + rc = pager_truncate(pPager, pSavepoint?pSavepoint->nOrig:pPager->dbOrigSize); assert( pPager->state>=PAGER_SHARED ); /* Now roll back all main journal file records that occur after byte @@ -2177,7 +2198,8 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ n /= pPager->pageSize; } if( pPager->state!=PAGER_UNLOCK ){ - pPager->dbSize = (int)n; + pPager->dbSize = (Pgno)n; + pPager->dbFileSize = (Pgno)n; pPager->dbSizeValid = 1; } } @@ -2248,7 +2270,11 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ } /* -** Truncate the file to the number of pages specified. +** Truncate the file to the number of pages specified. +** +** Unless an IO error occurs, this function is guaranteed to modify the +** database file itself. If an exclusive lock is not held when this function +** is called, one is obtained before truncating the file. */ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; @@ -2257,7 +2283,7 @@ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ sqlite3PagerPagecount(pPager, 0); if( pPager->errCode ){ rc = pPager->errCode; - }else if( nPagedbSize ){ + }else if( nPagedbFileSize ){ rc = syncJournal(pPager); if( rc==SQLITE_OK ){ /* Get an exclusive lock on the database before truncating. */ @@ -2271,6 +2297,36 @@ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ return rc; } +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Truncate the in-memory database file image to nPage pages. Unlike +** sqlite3PagerTruncate(), this function does not actually modify the +** database file on disk. It just sets the internal state of the pager +** object so that the truncation will be done when the current +** transaction is committed. +*/ +void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ + assert( pPager->dbSizeValid ); + assert( pPager->dbSize>=nPage ); + pPager->dbSize = nPage; +} + +/* +** Return the current size of the database file image in pages. This +** function differs from sqlite3PagerPagecount() in two ways: +** +** a) It may only be called when at least one reference to a database +** page is held. This guarantees that the database size is already +** known and a call to sqlite3OsFileSize() is not required. +** +** b) The return value is not adjusted for the locking page. +*/ +Pgno sqlite3PagerImageSize(Pager *pPager){ + assert( pPager->dbSizeValid ); + return pPager->dbSize; +} +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -2480,6 +2536,9 @@ static int pager_write_pagelist(PgHdr *pList){ if( pList->pgno==1 ){ memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); } + if( pList->pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pList->pgno; + } } #ifndef NDEBUG else{ @@ -3076,7 +3135,7 @@ static int pager_open_journal(Pager *pPager){ rc = pPager->errCode; goto failed_to_open_journal; } - pPager->origDbSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; rc = writeJournalHdr(pPager); @@ -3156,14 +3215,14 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){ ** overwritten with zeros. */ assert( pPager->nRec==0 ); - assert( pPager->origDbSize==0 ); + assert( pPager->dbOrigSize==0 ); assert( pPager->pInJournal==0 ); sqlite3PagerPagecount(pPager, 0); pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize ); if( !pPager->pInJournal ){ rc = SQLITE_NOMEM; }else{ - pPager->origDbSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; rc = writeJournalHdr(pPager); } } @@ -3223,7 +3282,7 @@ static int pager_write(PgHdr *pPg){ ** to the journal then we can return right away. */ sqlite3PcacheMakeDirty(pPg); - if( pageInJournal(pPg) && pageInSavepoint(pPg) ){ + if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ pPager->dirtyCache = 1; pPager->dbModified = 1; }else{ @@ -3254,7 +3313,7 @@ static int pager_write(PgHdr *pPg){ ** the transaction journal if it is not there already. */ if( !pageInJournal(pPg) && pPager->journalOpen ){ - if( pPg->pgno<=pPager->origDbSize ){ + if( pPg->pgno<=pPager->dbOrigSize ){ u32 cksum; char *pData2; @@ -3320,10 +3379,10 @@ static int pager_write(PgHdr *pPg){ ** the statement journal format differs from the standard journal format ** in that it omits the checksums and the header. */ - if( !pageInSavepoint(pPg) ){ + if( subjRequiresPage(pPg) ){ i64 offset = pPager->stmtNRec*(4+pPager->pageSize); char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7); - assert( pageInJournal(pPg) || pPg->pgno>pPager->origDbSize ); + assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); @@ -3488,12 +3547,12 @@ int sqlite3PagerDontWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; int rc; - if( pPg->pgno>pPager->origDbSize ){ + if( pPg->pgno>pPager->dbOrigSize ){ return SQLITE_OK; } if( pPager->pAlwaysRollback==0 ){ assert( pPager->pInJournal ); - pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->origDbSize); + pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->dbOrigSize); if( !pPager->pAlwaysRollback ){ return SQLITE_NOMEM; } @@ -3502,7 +3561,7 @@ int sqlite3PagerDontWrite(DbPage *pDbPage){ if( rc==SQLITE_OK && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ assert( pPager->state>=PAGER_SHARED ); - if( pPager->dbSize==pPg->pgno && pPager->origDbSizedbSize ){ + if( pPager->dbSize==pPg->pgno && pPager->dbOrigSizedbSize ){ /* If this pages is the last page in the file and the file has grown ** during the current transaction, then do NOT mark the page as clean. ** When the database file grows, we must make sure that the last page @@ -3545,14 +3604,14 @@ void sqlite3PagerDontRollback(DbPage *pPg){ */ if( pPager->journalOpen==0 || sqlite3BitvecTest(pPager->pAlwaysRollback, pPg->pgno) - || pPg->pgno>pPager->origDbSize + || pPg->pgno>pPager->dbOrigSize ){ return; } #ifdef SQLITE_SECURE_DELETE if( sqlite3BitvecTest(pPager->pInJournal, pPg->pgno)!=0 - || pPg->pgno>pPager->origDbSize ){ + || pPg->pgno>pPager->dbOrigSize ){ return; } #endif @@ -3567,7 +3626,7 @@ void sqlite3PagerDontRollback(DbPage *pPg){ ** pages on the freelist (ex: corrupt9.test) then the following is not ** necessarily true: */ - /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ); */ + /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->dbOrigSize ); */ assert( pPager->pInJournal!=0 ); sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); @@ -3611,6 +3670,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirect){ #ifdef SQLITE_ENABLE_ATOMIC_WRITE if( isDirect && pPager->fd->pMethods ){ const void *zBuf = pPgHdr->pData; + assert( pPager->dbFileSize>0 ); rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); } #endif @@ -3678,8 +3738,8 @@ int sqlite3PagerCommitPhaseOne( return SQLITE_OK; } - PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", - pPager->zFilename, zMaster, nTrunc); + PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", + pPager->zFilename, zMaster, pPager->dbSize); /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is a no-op. @@ -3705,7 +3765,7 @@ int sqlite3PagerCommitPhaseOne( !zMaster && pPager->journalOpen && pPager->journalOff==jrnlBufferSize(pPager) && - nTrunc==0 && + pPager->dbSize>=pPager->dbFileSize && (pPg==0 || pPg->pDirty==0) ); assert( pPager->journalOpen || pPager->journalMode==PAGER_JOURNALMODE_OFF ); @@ -3742,14 +3802,15 @@ int sqlite3PagerCommitPhaseOne( if( rc!=SQLITE_OK ) goto sync_exit; if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ #ifndef SQLITE_OMIT_AUTOVACUUM - if( nTrunc!=0 ){ + if( pPager->dbSizedbOrigSize ){ /* If this transaction has made the database smaller, then all pages ** being discarded by the truncation must be written to the journal ** file. */ Pgno i; Pgno iSkip = PAGER_MJ_PGNO(pPager); - for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){ + Pgno dbSize = pPager->dbSize; + for( i=pPager->dbSize+1; i<=pPager->dbOrigSize; i++ ){ if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ rc = sqlite3PagerGet(pPager, i, &pPg); if( rc!=SQLITE_OK ) goto sync_exit; @@ -3758,6 +3819,7 @@ int sqlite3PagerCommitPhaseOne( if( rc!=SQLITE_OK ) goto sync_exit; } } + pPager->dbSize = dbSize; } #endif rc = writeMasterJournal(pPager, zMaster); @@ -3768,8 +3830,8 @@ int sqlite3PagerCommitPhaseOne( if( rc!=SQLITE_OK ) goto sync_exit; #ifndef SQLITE_OMIT_AUTOVACUUM - if( nTrunc!=0 ){ - rc = sqlite3PagerTruncate(pPager, nTrunc); + if( pPager->dbSizedbFileSize ){ + rc = sqlite3PagerTruncate(pPager, pPager->dbSize); if( rc!=SQLITE_OK ) goto sync_exit; } #endif @@ -3797,8 +3859,8 @@ int sqlite3PagerCommitPhaseOne( IOTRACE(("DBSYNC %p\n", pPager)) pPager->state = PAGER_SYNCED; - }else if( MEMDB && nTrunc!=0 ){ - rc = sqlite3PagerTruncate(pPager, nTrunc); + }else if( MEMDB && pPager->dbSizedbFileSize ){ + rc = sqlite3PagerTruncate(pPager, pPager->dbSize); } sync_exit: @@ -4133,7 +4195,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; - assert( pageInJournal(pPg) || pPg->pgno>pPager->origDbSize ); + assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); assert( pPager->needSync ); } @@ -4182,7 +4244,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( pPager->needSync ); rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); if( rc!=SQLITE_OK ){ - if( pPager->pInJournal && needSyncPgno<=pPager->origDbSize ){ + if( pPager->pInJournal && needSyncPgno<=pPager->dbOrigSize ){ sqlite3BitvecClear(pPager->pInJournal, needSyncPgno); } return rc; diff --git a/src/pager.h b/src/pager.h index e69e7df1fc..03ce8f5cdd 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.89 2008/12/17 17:30:26 danielk1977 Exp $ +** @(#) $Id: pager.h,v 1.90 2008/12/27 15:23:13 danielk1977 Exp $ */ #ifndef _PAGER_H_ @@ -119,6 +119,11 @@ int sqlite3PagerSync(Pager *pPager); int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); +#ifndef SQLITE_OMIT_AUTOVACUUM + void sqlite3PagerTruncateImage(Pager*,Pgno); + Pgno sqlite3PagerImageSize(Pager *); +#endif + #ifdef SQLITE_HAS_CODEC void sqlite3PagerSetCodec(Pager*,void*(*)(void*,void*,Pgno,int),void*); #endif diff --git a/test/incrvacuum.test b/test/incrvacuum.test index 706d8b4164..4234a454fa 100644 --- a/test/incrvacuum.test +++ b/test/incrvacuum.test @@ -14,7 +14,7 @@ # Note: There are also some tests for incremental vacuum and IO # errors in incrvacuum_ioerr.test. # -# $Id: incrvacuum.test,v 1.20 2008/09/10 10:57:28 danielk1977 Exp $ +# $Id: incrvacuum.test,v 1.21 2008/12/27 15:23:13 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -732,6 +732,48 @@ do_test incrvacuum-14.1 { } db3 } {1 {file is encrypted or is not a database}} +do_test incrvacuum-15.1 { + db close + file delete -force test.db + sqlite3 db test.db + + set str [string repeat "abcdefghij" 500] + + execsql { + PRAGMA cache_size = 10; + PRAGMA auto_vacuum = incremental; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('a', $str); + INSERT INTO t1 VALUES('b', $str); + INSERT INTO t1 VALUES('c', $str); + INSERT INTO t1 VALUES('d', $str); + INSERT INTO t1 VALUES('e', $str); + INSERT INTO t1 VALUES('f', $str); + INSERT INTO t1 VALUES('g', $str); + INSERT INTO t1 VALUES('h', $str); + INSERT INTO t1 VALUES('i', $str); + INSERT INTO t1 VALUES('j', $str); + INSERT INTO t1 VALUES('j', $str); + + CREATE TABLE t2(x PRIMARY KEY, y); + INSERT INTO t2 VALUES('a', $str); + INSERT INTO t2 VALUES('b', $str); + INSERT INTO t2 VALUES('c', $str); + INSERT INTO t2 VALUES('d', $str); + + BEGIN; + DELETE FROM t2; + PRAGMA incremental_vacuum; + } + + catchsql {INSERT INTO t2 SELECT * FROM t1} + + execsql { + COMMIT; + PRAGMA integrity_check; + } +} {ok} + db2 close db3 close finish_test diff --git a/test/savepoint.test b/test/savepoint.test index 91c087d2db..17aefccdaf 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# $Id: savepoint.test,v 1.3 2008/12/23 11:46:28 danielk1977 Exp $ +# $Id: savepoint.test,v 1.4 2008/12/27 15:23:13 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -376,5 +376,98 @@ ifcapable {autovacuum && pragma} { integrity_check savepoint-6.4 } +#------------------------------------------------------------------------- +# The following tests, savepoint-7.*, attempt to break the logic +# surrounding savepoints by growing and shrinking the database file. +# +db close +file delete -force test.db +sqlite3 db test.db + +do_test savepoint-7.1 { + execsql { + PRAGMA auto_vacuum = incremental; + PRAGMA cache_size = 10; + BEGIN; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1(a) VALUES('alligator'); + INSERT INTO t1(a) VALUES('angelfish'); + INSERT INTO t1(a) VALUES('ant'); + INSERT INTO t1(a) VALUES('antelope'); + INSERT INTO t1(a) VALUES('ape'); + INSERT INTO t1(a) VALUES('baboon'); + INSERT INTO t1(a) VALUES('badger'); + INSERT INTO t1(a) VALUES('bear'); + INSERT INTO t1(a) VALUES('beetle'); + INSERT INTO t1(a) VALUES('bird'); + INSERT INTO t1(a) VALUES('bison'); + UPDATE t1 SET b = randstr(1000,1000); + UPDATE t1 SET b = b||randstr(1000,1000); + UPDATE t1 SET b = b||randstr(1000,1000); + UPDATE t1 SET b = b||randstr(10,1000); + COMMIT; + } + expr ([execsql { PRAGMA page_count }] > 20) +} {1} +do_test savepoint-7.2.1 { + execsql { + BEGIN; + SAVEPOINT one; + CREATE TABLE t2(a, b); + INSERT INTO t2 SELECT a, b FROM t1; + ROLLBACK TO one; + } + execsql { + PRAGMA integrity_check; + } +} {ok} +do_test savepoint-7.2.2 { + execsql { + COMMIT; + PRAGMA integrity_check; + } +} {ok} + +do_test savepoint-7.3.1 { + execsql { + CREATE TABLE t2(a, b); + INSERT INTO t2 SELECT a, b FROM t1; + } +} {} +do_test savepoint-7.3.2 { + execsql { + BEGIN; + SAVEPOINT one; + DELETE FROM t2; + PRAGMA incremental_vacuum; + SAVEPOINT two; + INSERT INTO t2 SELECT a, b FROM t1; + ROLLBACK TO two; + COMMIT; + } + execsql { PRAGMA integrity_check } +} {ok} + +do_test savepoint-7.4.1 { + db close + file delete -force test.db + sqlite3 db test.db + execsql { + PRAGMA auto_vacuum = incremental; + CREATE TABLE t1(a, b, PRIMARY KEY(a, b)); + INSERT INTO t1 VALUES(randstr(1000,1000), randstr(1000,1000)); + BEGIN; + DELETE FROM t1; + SAVEPOINT one; + PRAGMA incremental_vacuum; + ROLLBACK TO one; + COMMIT; + } + + execsql { PRAGMA integrity_check } +} {ok} + + + finish_test