From 41113b6429967b290386c05a4cbf6b9efc632b7a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Apr 2016 21:07:58 +0000 Subject: [PATCH 01/20] Defer opening the file used for the temp database (where CREATE TEMP TABLE tables are stored) until the database is too large to reside entirely within the cache. There are likely still problems on this branch. FossilOrigin-Name: be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f --- manifest | 20 +++--- manifest.uuid | 2 +- src/pager.c | 24 +++++-- src/pcache.c | 15 ++++- test/temptable2.test | 157 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 test/temptable2.test diff --git a/manifest b/manifest index 04532b4880..261df5d6bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Carry\stable\scolumn\stypes\sthrough\sinto\sVIEW\sdefinitions,\swhere\spossible. -D 2016-04-05T20:59:12.069 +C Defer\sopening\sthe\sfile\sused\sfor\sthe\stemp\sdatabase\s(where\sCREATE\sTEMP\sTABLE\stables\sare\sstored)\suntil\sthe\sdatabase\sis\stoo\slarge\sto\sreside\sentirely\swithin\sthe\scache.\sThere\sare\slikely\sstill\sproblems\son\sthis\sbranch. +D 2016-04-05T21:07:58.179 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,10 +362,10 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d +F src/pager.c 123dbae47bf27915a4b567915ff8dcc27cfcd369 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 -F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df +F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 @@ -1110,6 +1110,7 @@ F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 +F test/temptable2.test f3b198e386f6494082c87accc8780cdf84b36e6d F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1482,7 +1483,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0bf9926c7a7865694edd48535777248e73d86bbf -R c7f5a7683cc4a0d14362e20e6891847a -U drh -Z ec8e2cff7764d808576889c190bbf60c +P fb555c3c2af7f5e62ff839658f4fba7b645d3a68 +R f511e5baaeaad693345498b91b98e90f +T *branch * tempfiles-lazy-open +T *sym-tempfiles-lazy-open * +T -sym-trunk * +U dan +Z 505ba6148fb8f2c79237e86b07655c1c diff --git a/manifest.uuid b/manifest.uuid index d21138d9ad..0e8a9ef130 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb555c3c2af7f5e62ff839658f4fba7b645d3a68 \ No newline at end of file +be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c18b3a32f7..45f54ef30d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2006,7 +2006,11 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); + if( pPager->tempFile==0 || MEMDB ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ @@ -2374,9 +2378,13 @@ static int pager_playback_one_page( ** be written out into the database file before its journal file ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. + ** + ** Update: Another exception is for temp files that are not + ** in-memory databases. In this case the page may have been dirty + ** at the start of the transaction. */ assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); + if( pPager->tempFile==0 ) sqlite3PcacheMakeClean(pPg); } pager_set_pagehash(pPg); @@ -4258,8 +4266,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( pPager->tempFile==0 || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -5941,6 +5950,7 @@ int sqlite3PagerWrite(PgHdr *pPg){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -6179,11 +6189,11 @@ int sqlite3PagerCommitPhaseOne( /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); + if( pPager->tempFile ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ + ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -6845,7 +6855,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit && pPager->tempFile==0 ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); diff --git a/src/pcache.c b/src/pcache.c index 61e7587422..52661c28a9 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -254,7 +254,7 @@ sqlite3_pcache_page *sqlite3PcacheFetch( /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new -** page because new clean pages are available for reuse and the cache +** page because no clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to @@ -439,6 +439,17 @@ void sqlite3PcacheCleanAll(PCache *pCache){ } } +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -484,7 +495,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ + if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } diff --git a/test/temptable2.test b/test/temptable2.test new file mode 100644 index 0000000000..daeb206a29 --- /dev/null +++ b/test/temptable2.test @@ -0,0 +1,157 @@ +# 2016 March 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix temptable2 + +do_execsql_test 1.1 { + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); +} + +do_execsql_test 1.2 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100000 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM X; +} {} + +do_execsql_test 1.3 { + PRAGMA temp.integrity_check; +} {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.1 { + CREATE TEMP TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 2); +} {} + +do_execsql_test 2.2 { + BEGIN; + INSERT INTO t2 VALUES(3, 4); + SELECT * FROM t2; +} {1 2 3 4} + +do_execsql_test 2.3 { + ROLLBACK; + SELECT * FROM t2; +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t1; +} {1000} +do_execsql_test 3.1.2 { + BEGIN; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; + ROLLBACK; +} +do_execsql_test 3.1.3 { + SELECT count(*) FROM t1; +} {1000} +do_execsql_test 3.1.4 { PRAGMA temp.integrity_check } {ok} + +do_execsql_test 3.2.1 { + BEGIN; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; + SAVEPOINT abc; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==1; + ROLLBACK TO abc; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==2; + COMMIT; +} +do_execsql_test 3.2.2 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<10 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t1; + PRAGMA temp.page_count; +} {10 9} + +do_execsql_test 4.1.2 { + BEGIN; + UPDATE t1 SET b=randomblob(100); + ROLLBACK; +} + +do_execsql_test 4.1.3 { + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t2; + SELECT count(*) FROM t1; + PRAGMA temp.page_count; +} {500 10 292} + +do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, 2); + + PRAGMA temp.page_count; +} {286} + +do_execsql_test 5.1.2 { + BEGIN; + UPDATE t1 SET a=2; + UPDATE t2 SET a=randomblob(100); + SELECT count(*) FROM t1; + ROLLBACK; +} {1} + +do_execsql_test 5.1.3 { + UPDATE t2 SET a=randomblob(100); + + SELECT * FROM t1; +} {1 2} + +do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} + +finish_test + From d87efd7221c3a4edefe09bd2c18bf29b042ad542 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Apr 2016 15:39:03 +0000 Subject: [PATCH 02/20] Add tests to this branch. Fix a problem with temporary databases in auto-vacuum mode. FossilOrigin-Name: afe9bd9b4b5dc98dccf1bced80567515ab5c0117 --- manifest | 17 ++++++------- manifest.uuid | 2 +- src/pager.c | 8 +++--- test/temptable2.test | 58 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 261df5d6bf..0084c64a0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Defer\sopening\sthe\sfile\sused\sfor\sthe\stemp\sdatabase\s(where\sCREATE\sTEMP\sTABLE\stables\sare\sstored)\suntil\sthe\sdatabase\sis\stoo\slarge\sto\sreside\sentirely\swithin\sthe\scache.\sThere\sare\slikely\sstill\sproblems\son\sthis\sbranch. -D 2016-04-05T21:07:58.179 +C Add\stests\sto\sthis\sbranch.\sFix\sa\sproblem\swith\stemporary\sdatabases\sin\sauto-vacuum\smode. +D 2016-04-06T15:39:03.038 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 123dbae47bf27915a4b567915ff8dcc27cfcd369 +F src/pager.c 685f3cc08d045f64d20133144fb9a437914ea4c7 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -1110,7 +1110,7 @@ F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test f3b198e386f6494082c87accc8780cdf84b36e6d +F test/temptable2.test c16f96e996bf6f587a4df4199c62cf0ac9a1c0ea F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1483,10 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb555c3c2af7f5e62ff839658f4fba7b645d3a68 -R f511e5baaeaad693345498b91b98e90f -T *branch * tempfiles-lazy-open -T *sym-tempfiles-lazy-open * -T -sym-trunk * +P be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f +R 71d9b48ce9c812093890a1f227fcf740 U dan -Z 505ba6148fb8f2c79237e86b07655c1c +Z 749bb77cf880284a6ac38e76d6c94ae0 diff --git a/manifest.uuid b/manifest.uuid index 0e8a9ef130..f61492cdc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f \ No newline at end of file +afe9bd9b4b5dc98dccf1bced80567515ab5c0117 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 45f54ef30d..3d8df3cdbb 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6815,7 +6815,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - if( MEMDB ){ + if( pPager->tempFile ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -6855,7 +6855,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit && pPager->tempFile==0 ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit /* && pPager->tempFile==0 */ ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); @@ -6872,7 +6872,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ + if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -6889,7 +6889,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( MEMDB ){ + if( pPager->tempFile ){ assert( pPgOld ); sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); diff --git a/test/temptable2.test b/test/temptable2.test index daeb206a29..e702410f68 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -153,5 +153,63 @@ do_execsql_test 5.1.3 { do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} +#------------------------------------------------------------------------- +# Test this: +# +# 1. Page is DIRTY at the start of a transaction. +# 2. Page is written out as part of the transaction. +# 3. Page is then read back in. +# 4. Transaction is rolled back. Is the page now clean or dirty? +# +# This actually does work. Step 4 marks the page as clean. But it also +# writes to the database file itself. So marking it clean is correct - +# the page does match the contents of the db file. +# +reset_db + +do_execsql_test 6.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(x); + INSERT INTO t1 VALUES('one'); + + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; +} + +do_execsql_test 6.2 { + UPDATE t1 SET x='two'; -- step 1 + BEGIN; + UPDATE t2 SET a=randomblob(100); -- step 2 + SELECT * FROM t1; -- step 3 + ROLLBACK; -- step 4 + + SELECT count(*) FROM t2; + SELECT * FROM t1; +} {two 500 two} + +#------------------------------------------------------------------------- + +reset_db +sqlite3 db "" +do_execsql_test 7.1 { + PRAGMA auto_vacuum=INCREMENTAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(zeroblob(900)); + INSERT INTO t1 VALUES(zeroblob(900)); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + BEGIN; + DELETE FROM t1 WHERE rowid%2; + PRAGMA incremental_vacuum(4); + ROLLBACK; + PRAGMA integrity_check; +} {ok} + finish_test From 7f8def285efbf12c337050ac81788e38345909c6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Apr 2016 16:27:50 +0000 Subject: [PATCH 03/20] Fix a problem in test script lock.test. Fix minor issues in pager.c. FossilOrigin-Name: 84de8813c3b4007e3c7c3a286ce13020f2065c7b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 5 +++-- test/lock.test | 5 +++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0084c64a0f..fefd0eeca8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\sthis\sbranch.\sFix\sa\sproblem\swith\stemporary\sdatabases\sin\sauto-vacuum\smode. -D 2016-04-06T15:39:03.038 +C Fix\sa\sproblem\sin\stest\sscript\slock.test.\sFix\sminor\sissues\sin\spager.c. +D 2016-04-06T16:27:50.054 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 685f3cc08d045f64d20133144fb9a437914ea4c7 +F src/pager.c 2047e837a5df26c16271e77e6b30021d0cdc98b7 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -877,7 +877,7 @@ F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 +F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P be5a549eba6cf8e29cb6b9824fd6d0db9d03ca7f -R 71d9b48ce9c812093890a1f227fcf740 +P afe9bd9b4b5dc98dccf1bced80567515ab5c0117 +R 35c99f2c80a6bf5e586b897ba6d7e956 U dan -Z 749bb77cf880284a6ac38e76d6c94ae0 +Z 67c50ed57a780aa24194c52733684bdf diff --git a/manifest.uuid b/manifest.uuid index f61492cdc3..832abb0052 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afe9bd9b4b5dc98dccf1bced80567515ab5c0117 \ No newline at end of file +84de8813c3b4007e3c7c3a286ce13020f2065c7b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3d8df3cdbb..c1c1134047 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6855,7 +6855,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit /* && pPager->tempFile==0 */ ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); @@ -7142,7 +7142,8 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); } #endif diff --git a/test/lock.test b/test/lock.test index f785c4bb23..534aa3b9a4 100644 --- a/test/lock.test +++ b/test/lock.test @@ -423,8 +423,9 @@ do_test lock-6.5 { # * there exists one or more active read-only statements, and # * a transaction that modified zero database pages is committed. # -set temp_status unlocked -if {$TEMP_STORE>=2} {set temp_status unknown} +#set temp_status unlocked +#if {$TEMP_STORE>=2} {set temp_status unknown} +set temp_status unknown do_test lock-7.1 { set STMT [sqlite3_prepare $DB "SELECT * FROM sqlite_master" -1 TAIL] sqlite3_step $STMT From 9131ab93865d961f6408800f969e3996d78db34e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Apr 2016 18:20:51 +0000 Subject: [PATCH 04/20] For a pager backed by a temp file, store the main journal in memory until it is at least sqlite3_config.nStmtSpill bytes in size. Prevent the backup API from being used to change the page-size of a temp file. FossilOrigin-Name: 84c557010c211595d2ec80b62c63af1c7f4714bd --- manifest | 18 +++++------ manifest.uuid | 2 +- src/pager.c | 30 +++++++++--------- src/test3.c | 6 ++-- test/pragma.test | 15 ++++++++- test/temptable2.test | 74 +++++++++++++++++++++++++++++++++++++++----- 6 files changed, 110 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index fefd0eeca8..103a4b00bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\stest\sscript\slock.test.\sFix\sminor\sissues\sin\spager.c. -D 2016-04-06T16:27:50.054 +C For\sa\spager\sbacked\sby\sa\stemp\sfile,\sstore\sthe\smain\sjournal\sin\smemory\suntil\sit\sis\sat\sleast\ssqlite3_config.nStmtSpill\sbytes\sin\ssize.\sPrevent\sthe\sbackup\sAPI\sfrom\sbeing\sused\sto\schange\sthe\spage-size\sof\sa\stemp\sfile. +D 2016-04-06T18:20:51.565 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 2047e837a5df26c16271e77e6b30021d0cdc98b7 +F src/pager.c 91d7462aa2429e30d64c705f75ba36cace90c241 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -387,7 +387,7 @@ F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 56569acc73d36e836b64aefecbbb709a92ba0077 F src/test1.c 7187b7e924bfc97780e6fd2a40dad94a32bddca0 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b -F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f +F src/test3.c 0df6f8dbb4cbaa7106397c70a271fa6a43659042 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 @@ -976,7 +976,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test cd1fa041074ed08eeaa563e4d1bacb0c69337ec1 -F test/pragma.test afbf028be1c35b68f57db8eb015c4a3c59d8f28e +F test/pragma.test 36daccdbfbdd958a5f65c034141ab2bfd959808e F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 3f1984a04657331f838df5c519b443c2088df922 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc @@ -1110,7 +1110,7 @@ F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test c16f96e996bf6f587a4df4199c62cf0ac9a1c0ea +F test/temptable2.test 707c01c00b4a8fd050e0db5dcef448e998b9ddb5 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afe9bd9b4b5dc98dccf1bced80567515ab5c0117 -R 35c99f2c80a6bf5e586b897ba6d7e956 +P 84de8813c3b4007e3c7c3a286ce13020f2065c7b +R 54ab030014fa87b230c7ab4f5fdc139b U dan -Z 67c50ed57a780aa24194c52733684bdf +Z dbab9fc4f54135fb156cdc6ccf0d8253 diff --git a/manifest.uuid b/manifest.uuid index 832abb0052..515a3c2874 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84de8813c3b4007e3c7c3a286ce13020f2065c7b \ No newline at end of file +84c557010c211595d2ec80b62c63af1c7f4714bd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c1c1134047..79dbaec5c2 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1170,6 +1170,8 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#else +# define jrnlBufferSize(x) 0 #endif /* @@ -5562,24 +5564,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -6532,10 +6534,10 @@ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ } /* -** Return true if this is an in-memory pager. +** Return true if this is an in-memory or temp-file backed pager. */ int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; + return pPager->tempFile; } /* diff --git a/src/test3.c b/src/test3.c index 2a41068e5f..817e8a88fd 100644 --- a/src/test3.c +++ b/src/test3.c @@ -547,7 +547,7 @@ static int btree_from_db( /* ** Usage: btree_ismemdb ID ** -** Return true if the B-Tree is in-memory. +** Return true if the B-Tree is currently stored entirely in memory. */ static int btree_ismemdb( void *NotUsed, @@ -557,6 +557,7 @@ static int btree_ismemdb( ){ Btree *pBt; int res; + sqlite3_file *pFile; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], @@ -566,7 +567,8 @@ static int btree_ismemdb( pBt = sqlite3TestTextToPtr(argv[1]); sqlite3_mutex_enter(pBt->db->mutex); sqlite3BtreeEnter(pBt); - res = sqlite3PagerIsMemdb(sqlite3BtreePager(pBt)); + pFile = sqlite3PagerFile(sqlite3BtreePager(pBt)); + res = (pFile->pMethods==0); sqlite3BtreeLeave(pBt); sqlite3_mutex_leave(pBt->db->mutex); Tcl_SetObjResult(interp, Tcl_NewBooleanObj(res)); diff --git a/test/pragma.test b/test/pragma.test index acbe74599e..2c73090551 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1083,7 +1083,20 @@ do_test pragma-8.2.15 { # "memory" or "disk" as appropriate. # proc check_temp_store {} { - db eval {CREATE TEMP TABLE IF NOT EXISTS a(b)} + db eval { + PRAGMA temp.cache_size = 1; + CREATE TEMP TABLE IF NOT EXISTS a(b); + DELETE FROM a; + INSERT INTO a VALUES(randomblob(1000)); + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + } db eval {PRAGMA database_list} { if {$name=="temp"} { set bt [btree_from_db db 1] diff --git a/test/temptable2.test b/test/temptable2.test index e702410f68..e87ffa89d9 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -113,8 +113,12 @@ do_execsql_test 4.1.3 { SELECT count(*) FROM t2; SELECT count(*) FROM t1; - PRAGMA temp.page_count; -} {500 10 292} +} {500 10} + +do_test 4.1.4 { + set n [db one { PRAGMA temp.page_count }] + expr ($n >280 && $n < 300) +} 1 do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok} @@ -133,11 +137,14 @@ do_execsql_test 5.1.1 { CREATE TEMP TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); INSERT INTO t1 VALUES(1, 2); +} - PRAGMA temp.page_count; -} {286} +do_test 5.1.2 { + set n [db one { PRAGMA temp.page_count }] + expr ($n > 280 && $n < 290) +} {1} -do_execsql_test 5.1.2 { +do_execsql_test 5.1.3 { BEGIN; UPDATE t1 SET a=2; UPDATE t2 SET a=randomblob(100); @@ -145,13 +152,13 @@ do_execsql_test 5.1.2 { ROLLBACK; } {1} -do_execsql_test 5.1.3 { +do_execsql_test 5.1.4 { UPDATE t2 SET a=randomblob(100); SELECT * FROM t1; } {1 2} -do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} +do_execsql_test 5.1.5 { PRAGMA temp.integrity_check } {ok} #------------------------------------------------------------------------- # Test this: @@ -192,7 +199,7 @@ do_execsql_test 6.2 { } {two 500 two} #------------------------------------------------------------------------- - +# reset_db sqlite3 db "" do_execsql_test 7.1 { @@ -211,5 +218,56 @@ do_execsql_test 7.1 { PRAGMA integrity_check; } {ok} +#------------------------------------------------------------------------- +# Try changing the page size using a backup operation when pages are +# stored in main-memory only. +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + PRAGMA page_count; +} {13} + +do_test 8.2 { + sqlite3 tmp "" + execsql { + PRAGMA page_size = 8192; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + PRAGMA page_count; + } tmp +} {10} + +do_test 8.3 { + sqlite3_backup B tmp main db main + B step 5 + B finish +} {SQLITE_READONLY} + +do_test 8.4 { + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + PRAGMA page_size; + } tmp +} {100 ok 8192} + +do_test 8.5 { + tmp eval { UPDATE t1 SET a=randomblob(100) } +} {} + +do_test 8.6 { + sqlite3_backup B tmp main db main + B step 1000 + B finish +} {SQLITE_READONLY} + +tmp close + finish_test From 67330a1224271e0186445c8f68754b214991df3a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 18:07:47 +0000 Subject: [PATCH 05/20] Fixes for OOM and IO error handling with temp file databases. FossilOrigin-Name: 4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 --- manifest | 15 +++--- manifest.uuid | 2 +- src/pager.c | 15 +++--- test/cffault.test | 2 +- test/tempfault.test | 114 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 test/tempfault.test diff --git a/manifest b/manifest index 2cc5625474..1c7823a2ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthis\sbranch\swith\sthe\slatest\schanges\sfrom\sthe\strunk. -D 2016-04-11T09:39:25.021 +C Fixes\sfor\sOOM\sand\sIO\serror\shandling\swith\stemp\sfile\sdatabases. +D 2016-04-11T18:07:47.205 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bde4844f0849cab5924c6a81178f8500774ce76b F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 91d7462aa2429e30d64c705f75ba36cace90c241 +F src/pager.c 4cc8a5b3979db65725eaa03650a6d54219122b8d F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 52cdeb4f37634d0ccd2998aab099b7bbb690b0d3 F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -551,7 +551,7 @@ F test/capi3c.test 0b9edb0c2156a964b9271cd5ea7ae56736cc2fcb F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 -F test/cffault.test aadc1f61f8811cb600e3e069acbf8796f472a096 +F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 @@ -1109,6 +1109,7 @@ F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 +F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 707c01c00b4a8fd050e0db5dcef448e998b9ddb5 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 @@ -1483,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84c557010c211595d2ec80b62c63af1c7f4714bd ef1966c2469a0f5dbdb31a0287bd37badb2b8f28 -R 95ac2720b81ae4f698b1e227fe8fc71b +P 982b753d0b6a3ed9fba33ed41523b2cd42280276 +R 46e56336ea4184d65ce3043234f11f03 U dan -Z 4e2c4b80dd3173e2cb79a80f72f7a7a8 +Z 58afa17a4e51610cdca345488b854923 diff --git a/manifest.uuid b/manifest.uuid index 9b0cbf0915..9e8a4101e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -982b753d0b6a3ed9fba33ed41523b2cd42280276 \ No newline at end of file +4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 79dbaec5c2..9baddc2986 100644 --- a/src/pager.c +++ b/src/pager.c @@ -958,7 +958,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); break; } @@ -1819,11 +1819,14 @@ static void pager_unlock(Pager *pPager){ ** trusted. Now that there are no outstanding references to the pager, ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. - */ - if( pPager->errCode ){ - assert( !MEMDB ); + ** + ** Exception: There is no way out of the error state for temp files. + ** This is because it is not possible to call pager_reset() on a temp + ** file pager (as this may discard the only copy of some data). */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); + if( pPager->tempFile==0 && pPager->errCode ){ pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; + pPager->changeCountDone = 0; pPager->eState = PAGER_OPEN; pPager->errCode = SQLITE_OK; if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); @@ -5034,8 +5037,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); + if( pPager->tempFile && pPager->errCode ) { return pPager->errCode; } assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ diff --git a/test/cffault.test b/test/cffault.test index 79cefd24c4..0d029ece37 100644 --- a/test/cffault.test +++ b/test/cffault.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix cacheflush +set testprefix cffault source $testdir/malloc_common.tcl # Run the supplied SQL on a copy of the database currently stored on diff --git a/test/tempfault.test b/test/tempfault.test new file mode 100644 index 0000000000..da1ae12375 --- /dev/null +++ b/test/tempfault.test @@ -0,0 +1,114 @@ +# 2016 April 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for fault-injection when SQLite is used with +# a temp file database. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix tempfault + +sqlite3_memdebug_vfs_oom_test 0 + +do_faultsim_test 1 -faults oom* -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } +} -body { + execsql { INSERT INTO t1 VALUES(5, 6) } +} -test { + faultsim_test_result {0 {}} + set rc [catch { execsql { SELECT * FROM t1 } } msg] + if {$rc==0 && $msg != "1 2 3 4 5 6" && $msg != "1 2 3 4"} { + error "data mismatch 1: $msg" + } + if {$testrc==0 && $msg != "1 2 3 4 5 6"} { + error "data mismatch 2: $msg" + } + faultsim_integrity_check +} + +do_faultsim_test 2 -faults oom* -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { UPDATE t1 SET a = randomblob(99) } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 3 -faults oom* -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET a = randomblob(99); + SAVEPOINT abc; + UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0; + ROLLBACK TO abc; + UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0; + ROLLBACK TO abc; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 4 -faults * -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET a = randomblob(99); + SAVEPOINT abc; + UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0; + ROLLBACK TO abc; + UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0; + ROLLBACK TO abc; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} +} + +sqlite3_memdebug_vfs_oom_test 1 +finish_test From b9f11f932dd6b924d5707222c43a72ea2291464a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 18:49:37 +0000 Subject: [PATCH 06/20] Add extra tests to temptable2.test. FossilOrigin-Name: 7dd9d4c15bb6b6ef9127dc7c80a189c55e714dc9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 1c7823a2ff..58157eb4f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sOOM\sand\sIO\serror\shandling\swith\stemp\sfile\sdatabases. -D 2016-04-11T18:07:47.205 +C Add\sextra\stests\sto\stemptable2.test. +D 2016-04-11T18:49:37.527 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test 707c01c00b4a8fd050e0db5dcef448e998b9ddb5 +F test/temptable2.test f04c411560dadebcc44aba631c602c908dfe1dd8 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 982b753d0b6a3ed9fba33ed41523b2cd42280276 -R 46e56336ea4184d65ce3043234f11f03 +P 4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 +R d2dfbf56b62d35c4cf496af1896eae37 U dan -Z 58afa17a4e51610cdca345488b854923 +Z db78b11117984d67b6a3aa7fe90754f9 diff --git a/manifest.uuid b/manifest.uuid index 9e8a4101e2..66abf69ec2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4eb06e843af60d5e533793618c6e0e9b7ef7a1a6 \ No newline at end of file +7dd9d4c15bb6b6ef9127dc7c80a189c55e714dc9 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index e87ffa89d9..af566ab529 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -269,5 +269,42 @@ do_test 8.6 { tmp close +#------------------------------------------------------------------------- +# Try inserts and deletes with a large db in auto-vacuum mode. Check +# +reset_db +do_execsql_test 9.1 { + PRAGMA cache_size = 15; + CREATE TABLE tx(a, b); + CREATE INDEX i1 ON tx(a); + CREATE INDEX i2 ON tx(b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; +} +for {set i 2} {$i<20} {incr i} { + + do_execsql_test 9.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } + + do_execsql_test 9.$i.2 { PRAGMA integrity_check } ok + + do_execsql_test 9.$i.3 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + } + + do_execsql_test 9.$i.4 { PRAGMA integrity_check } ok + + do_execsql_test 9.$i.5 { + BEGIN; + DELETE FROM tx WHERE (random()%3)==0; + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + COMMIT; + } + + do_execsql_test 9.$i.6 { PRAGMA integrity_check } ok +} + finish_test From b5a2592a9e1625b84b975d2bfbd2b8587bf58185 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 19:23:36 +0000 Subject: [PATCH 07/20] Add tests for wal mode to temptable2.test. FossilOrigin-Name: c6d0d441a1ab4873caedf2f23543444cea5d7f5a --- manifest | 12 +++++----- manifest.uuid | 2 +- test/temptable2.test | 55 ++++++++++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index d5aafb59c4..017d301d54 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2016-04-11T18:50:25.500 +C Add\stests\sfor\swal\smode\sto\stemptable2.test. +D 2016-04-11T19:23:36.940 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test f04c411560dadebcc44aba631c602c908dfe1dd8 +F test/temptable2.test 79a30edb79c4d5a4916f208f97935a70efd4b3ba F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7dd9d4c15bb6b6ef9127dc7c80a189c55e714dc9 2764aeaa11f38cf2ff4d6191e6d5466ddb203022 -R 7b4c1f71edaf782a86682a72496a355b +P ffc62af1d503c6e7fc0059d9c2ee57a3bef9dd32 +R b5aaac24011d4d477c13603957856121 U dan -Z 4c9d4d3f9ec993332df2230efd24eb4f +Z 91cb540a3f9a28d737c21f7feeaff7cc diff --git a/manifest.uuid b/manifest.uuid index 0dd130d179..fa8dceac08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffc62af1d503c6e7fc0059d9c2ee57a3bef9dd32 \ No newline at end of file +c6d0d441a1ab4873caedf2f23543444cea5d7f5a \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index af566ab529..44a148ddfb 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -272,38 +272,47 @@ tmp close #------------------------------------------------------------------------- # Try inserts and deletes with a large db in auto-vacuum mode. Check # -reset_db -do_execsql_test 9.1 { - PRAGMA cache_size = 15; - CREATE TABLE tx(a, b); - CREATE INDEX i1 ON tx(a); - CREATE INDEX i2 ON tx(b); +foreach {tn mode} { + 1 delete + 2 wal +} { + reset_db + do_execsql_test 9.$tn.1.1 { + PRAGMA cache_size = 15; + PRAGMA auto_vacuum = 1; + } + do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" $mode - WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) - INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; -} -for {set i 2} {$i<20} {incr i} { - - do_execsql_test 9.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } - - do_execsql_test 9.$i.2 { PRAGMA integrity_check } ok - - do_execsql_test 9.$i.3 { - WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + do_execsql_test 9.$tn.1.3 { + CREATE TABLE tx(a, b); + CREATE INDEX i1 ON tx(a); + CREATE INDEX i2 ON tx(b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; } - do_execsql_test 9.$i.4 { PRAGMA integrity_check } ok + for {set i 2} {$i<20} {incr i} { + do_execsql_test 9.$tn.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } - do_execsql_test 9.$i.5 { - BEGIN; + do_execsql_test 9.$tn.$i.2 { PRAGMA integrity_check } ok + + do_execsql_test 9.$tn.$i.3 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + } + + do_execsql_test 9.$tn.$i.4 { PRAGMA integrity_check } ok + + do_execsql_test 9.$tn.$i.5 { + BEGIN; DELETE FROM tx WHERE (random()%3)==0; WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; - COMMIT; - } + COMMIT; + } - do_execsql_test 9.$i.6 { PRAGMA integrity_check } ok + do_execsql_test 9.$tn.$i.6 { PRAGMA integrity_check } ok + } } finish_test From 0e55da2ef8999a44fe094b4f385423b1b60b304d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Apr 2016 19:24:56 +0000 Subject: [PATCH 08/20] Fix a typo in temptable2.test. FossilOrigin-Name: 04b1890fbc19eb3fa935083c1664fbbfb67dad93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 017d301d54..6d6c99e5a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\swal\smode\sto\stemptable2.test. -D 2016-04-11T19:23:36.940 +C Fix\sa\stypo\sin\stemptable2.test. +D 2016-04-11T19:24:56.197 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1f123a0757f6f04f0341accb46457e116817159a @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test 79a30edb79c4d5a4916f208f97935a70efd4b3ba +F test/temptable2.test e62783549be26283e1b3725f0de3309411be6c84 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffc62af1d503c6e7fc0059d9c2ee57a3bef9dd32 -R b5aaac24011d4d477c13603957856121 +P c6d0d441a1ab4873caedf2f23543444cea5d7f5a +R 24eeba37ecde224a7caa9038421064fb U dan -Z 91cb540a3f9a28d737c21f7feeaff7cc +Z 162e6a9a240e1cae98b0ba71f957c9d5 diff --git a/manifest.uuid b/manifest.uuid index fa8dceac08..3e7e5faf24 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6d0d441a1ab4873caedf2f23543444cea5d7f5a \ No newline at end of file +04b1890fbc19eb3fa935083c1664fbbfb67dad93 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index 44a148ddfb..c5e53de635 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -277,11 +277,12 @@ foreach {tn mode} { 2 wal } { reset_db + sqlite3 db "" do_execsql_test 9.$tn.1.1 { PRAGMA cache_size = 15; PRAGMA auto_vacuum = 1; } - do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" $mode + do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" delete do_execsql_test 9.$tn.1.3 { CREATE TABLE tx(a, b); From 199f56b98455731ccfd8297bf6f3906866c8309c Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Apr 2016 19:09:29 +0000 Subject: [PATCH 09/20] Once a temporary database file has been opened, flush all dirty pages to disk when comitting a transaction. FossilOrigin-Name: bbac71aa2aa2380d393cda3be64b0208b464b27e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6c3dd2f0cc..1f71471219 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk. -D 2016-04-12T16:10:10.308 +C Once\sa\stemporary\sdatabase\sfile\shas\sbeen\sopened,\sflush\sall\sdirty\spages\sto\sdisk\swhen\scomitting\sa\stransaction. +D 2016-04-12T19:09:29.339 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bde4844f0849cab5924c6a81178f8500774ce76b F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 4cc8a5b3979db65725eaa03650a6d54219122b8d +F src/pager.c 525f3698f9a6b47a6e0f5496675529a183c2462b F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04b1890fbc19eb3fa935083c1664fbbfb67dad93 7e7289655185e7643ead6d685922528bc4d9e0ae -R 5d35ce920fcfc7013551c54ce7d1cde0 -U drh -Z 97342c74358cb773c29965a93119c3f8 +P 9682c0433c04713c28bd9105a7e20af7372f873e +R ade8e2f9d92ad43fcc4c6c4e5d1634ec +U dan +Z 4a91ff9d1af5229ff41acb5284c79d70 diff --git a/manifest.uuid b/manifest.uuid index 01a6eda77e..90747752ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9682c0433c04713c28bd9105a7e20af7372f873e \ No newline at end of file +bbac71aa2aa2380d393cda3be64b0208b464b27e \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 9baddc2986..e8c612e5c8 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2011,7 +2011,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( pPager->tempFile==0 || MEMDB ){ + if( isOpen(pPager->fd) || MEMDB ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); @@ -4273,7 +4273,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ assert( !pagerUseWal(pPager) ); assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); - assert( pPager->tempFile==0 || pList->pDirty==0 ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -6195,7 +6195,8 @@ int sqlite3PagerCommitPhaseOne( if( pPager->eStatetempFile ); - if( pPager->tempFile ){ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( !isOpen(pPager->fd) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any ** backup in progress needs to be restarted. */ From 0f52455a357769627eaaab4575dd8f5760b729a0 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Apr 2016 16:52:11 +0000 Subject: [PATCH 10/20] On commit, flush dirty temp-file pages only if the file is already open and 25% or more of the cache is dirty. FossilOrigin-Name: f6babf2920340f25815c0a3c58de1e902c2f5542 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/pager.c | 21 ++++++++++++++++++++- src/pcache.c | 11 +++++++++++ src/pcache.h | 3 +++ 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1f71471219..0105dfb0f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Once\sa\stemporary\sdatabase\sfile\shas\sbeen\sopened,\sflush\sall\sdirty\spages\sto\sdisk\swhen\scomitting\sa\stransaction. -D 2016-04-12T19:09:29.339 +C On\scommit,\sflush\sdirty\stemp-file\spages\sonly\sif\sthe\sfile\sis\salready\sopen\sand\s25%\sor\smore\sof\sthe\scache\sis\sdirty. +D 2016-04-13T16:52:11.775 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,11 +362,11 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bde4844f0849cab5924c6a81178f8500774ce76b F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 525f3698f9a6b47a6e0f5496675529a183c2462b +F src/pager.c 4d849ad718980d698157cd136a40dc91cbeff4d3 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e -F src/pcache.c e9c00846d3dcdaa75b288c6f16238c2fe2177823 -F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 +F src/pcache.c d63b34cce0a8aba1fa552428b2790e13877db553 +F src/pcache.h 60bc9893bfc0e16f8178fb5d8b6fcb8fab1d93c0 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c @@ -1484,7 +1484,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9682c0433c04713c28bd9105a7e20af7372f873e -R ade8e2f9d92ad43fcc4c6c4e5d1634ec +P bbac71aa2aa2380d393cda3be64b0208b464b27e +R d455655e720096e5ec7c439c248aa874 +T *branch * tempfiles-25 +T *sym-tempfiles-25 * +T -sym-tempfiles-lazy-open * U dan -Z 4a91ff9d1af5229ff41acb5284c79d70 +Z 1d78549afb34b97c4ff4b3e2f6fd7c59 diff --git a/manifest.uuid b/manifest.uuid index 90747752ed..5f347a440e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbac71aa2aa2380d393cda3be64b0208b464b27e \ No newline at end of file +f6babf2920340f25815c0a3c58de1e902c2f5542 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e8c612e5c8..2530a2836e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1873,6 +1873,25 @@ static int pager_error(Pager *pPager, int rc){ static int pager_truncate(Pager *pPager, Pgno nPage); +/* +** The write transaction open on the pager passed as the only argument is +** being committed. This function returns true if all dirty pages should +** be flushed to disk, or false otherwise. Pages should be flushed to disk +** unless one of the following is true: +** +** * The db is an in-memory database. +** +** * The db is a temporary database and the db file has not been opened. +** +** * The db is a temporary database and the cache contains less than +** C/4 dirty pages, where C is the configured cache-size. +*/ +static int pagerFlushOnCommit(Pager *pPager){ + if( pPager->tempFile==0 ) return 1; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -2011,7 +2030,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( isOpen(pPager->fd) || MEMDB ){ + if( pagerFlushOnCommit(pPager) ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); diff --git a/src/pcache.c b/src/pcache.c index 52661c28a9..561672b9ce 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -686,6 +686,17 @@ void sqlite3PcacheShrink(PCache *pCache){ */ int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } +/* +** Return the number of dirty pages currently in the cache, as a percentage +** of the configured cache size. +*/ +int sqlite3PCachePercentDirty(PCache *pCache){ + PgHdr *pDirty; + int nDirty = 0; + int nCache = numberOfCachePages(pCache); + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return (int)(((i64)nDirty * 100) / nCache); +} #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* diff --git a/src/pcache.h b/src/pcache.h index 475c04c061..0caf605ff8 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -173,4 +173,7 @@ void sqlite3PCacheSetDefault(void); int sqlite3HeaderSizePcache(void); int sqlite3HeaderSizePcache1(void); +/* Number of dirty pages as a percentage of the configured cache size */ +int sqlite3PCachePercentDirty(PCache*); + #endif /* _PCACHE_H_ */ From b5895e50f980b7cb41c68073902bcfe1e4f81168 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 18 Apr 2016 13:30:50 +0000 Subject: [PATCH 11/20] sqlite3PCachePerecentDirty() should return 0 if the pcache is empty. FossilOrigin-Name: 85f3d80eb7319754792eef76996b2c740a2b0073 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index acc14071dc..4b67df4189 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk\svia\stempfiles-lazy-open. -D 2016-04-13T19:20:23.336 +C sqlite3PCachePerecentDirty()\sshould\sreturn\s0\sif\sthe\spcache\sis\sempty. +D 2016-04-18T13:30:50.144 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -365,7 +365,7 @@ F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 4d849ad718980d698157cd136a40dc91cbeff4d3 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e -F src/pcache.c d63b34cce0a8aba1fa552428b2790e13877db553 +F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 F src/pcache.h 60bc9893bfc0e16f8178fb5d8b6fcb8fab1d93c0 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f6babf2920340f25815c0a3c58de1e902c2f5542 e2edd34e79af906e7069e3e837730b12de0b73fe -R ae2dffba3d0e430c251b8372421692c3 +P ae16310c4e9a9b7e94874bd912fc6a4324cfaa2b +R 0620b8f840563243b73e6076ebdcd81f U drh -Z 2aed413a070df399f71581c0517ce593 +Z 04d9a6e624a4591b3aa3f8ec5734f1ff diff --git a/manifest.uuid b/manifest.uuid index de9f4ee7ee..9ac43a726e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae16310c4e9a9b7e94874bd912fc6a4324cfaa2b \ No newline at end of file +85f3d80eb7319754792eef76996b2c740a2b0073 \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 561672b9ce..f700c2ff66 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -695,7 +695,7 @@ int sqlite3PCachePercentDirty(PCache *pCache){ int nDirty = 0; int nCache = numberOfCachePages(pCache); for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; - return (int)(((i64)nDirty * 100) / nCache); + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; } #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) From 6bcfe8b61f6b9390622670de373359159fe06d44 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Apr 2016 15:24:46 +0000 Subject: [PATCH 12/20] Add a function prototype in order to fix a compiler warning. FossilOrigin-Name: 49aec9718d61c9f12aed96f530128666c3b01c81 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.h | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5048173d7c..8a5fd27015 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\supdates\sfrom\strunk. -D 2016-04-21T15:03:37.535 +C Add\sa\sfunction\sprototype\sin\sorder\sto\sfix\sa\scompiler\swarning. +D 2016-04-21T15:24:46.372 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -366,7 +366,7 @@ F src/pager.c bb8c7ff706aa88e7c2ec9a6175a128c01bc3ae5f F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 -F src/pcache.h 60bc9893bfc0e16f8178fb5d8b6fcb8fab1d93c0 +F src/pcache.h 6b865be765d1ebd06145219550b10921c7da7cc9 F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9768de502e34da42f3ad955947c23da50f57bce c4f165c460c4244ed434107feac005efa3c386cf -R 46a947ce1c80cd48530d3e6dff00dfca +P cc28106e5c196c0a9678fbbd06f6afd49271a8e8 +R c97bc8da233adc2bbb054b217c885bd5 U drh -Z 049109eccf475c334bc4eb599a0e0693 +Z fa8f1e96dc1996c819bff77f7b75b34d diff --git a/manifest.uuid b/manifest.uuid index 8cdc841c3b..31bd76e737 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc28106e5c196c0a9678fbbd06f6afd49271a8e8 \ No newline at end of file +49aec9718d61c9f12aed96f530128666c3b01c81 \ No newline at end of file diff --git a/src/pcache.h b/src/pcache.h index 0caf605ff8..04bce289d0 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -99,6 +99,7 @@ void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ void sqlite3PcacheMove(PgHdr*, Pgno); From 6572c16ae1a1f554475c4993eb0c753a2c5cf8d9 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Apr 2016 14:55:28 +0000 Subject: [PATCH 13/20] Fix some problems to do with temp-file databases and recovering from IO and SQLITE_FULL errors. FossilOrigin-Name: 3d61da4a76af8c9c2a293df085f3ed5a7bb447df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 26 +++++++++++++++----------- test/tempfault.test | 30 +++++++++++++++++++++++++----- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 8a5fd27015..ec3e51504b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfunction\sprototype\sin\sorder\sto\sfix\sa\scompiler\swarning. -D 2016-04-21T15:24:46.372 +C Fix\ssome\sproblems\sto\sdo\swith\stemp-file\sdatabases\sand\srecovering\sfrom\sIO\sand\sSQLITE_FULL\serrors. +D 2016-04-23T14:55:28.020 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c bb8c7ff706aa88e7c2ec9a6175a128c01bc3ae5f +F src/pager.c 4981dc6154ce111361a7cd35eaa26aadf8914ea3 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1109,7 +1109,7 @@ F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 -F test/tempfault.test 0c80f23c651cf6452c27cacf09f4c2ee9527a912 +F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test e62783549be26283e1b3725f0de3309411be6c84 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cc28106e5c196c0a9678fbbd06f6afd49271a8e8 -R c97bc8da233adc2bbb054b217c885bd5 -U drh -Z fa8f1e96dc1996c819bff77f7b75b34d +P 49aec9718d61c9f12aed96f530128666c3b01c81 +R 98471ae1c4ed8f399ba91e3e119d0ef8 +U dan +Z 4115a15d29a4b7f5f76b2491e48cc4f1 diff --git a/manifest.uuid b/manifest.uuid index 31bd76e737..7b1996a745 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49aec9718d61c9f12aed96f530128666c3b01c81 \ No newline at end of file +3d61da4a76af8c9c2a293df085f3ed5a7bb447df \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 41d3297e29..a749b8a6b3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1824,12 +1824,16 @@ static void pager_unlock(Pager *pPager){ ** This is because it is not possible to call pager_reset() on a temp ** file pager (as this may discard the only copy of some data). */ assert( pPager->errCode==SQLITE_OK || !MEMDB ); - if( pPager->tempFile==0 && pPager->errCode ){ - pager_reset(pPager); - pPager->changeCountDone = 0; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; + if( pPager->errCode ){ + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; } pPager->journalOff = 0; @@ -2319,7 +2323,7 @@ static int pager_playback_one_page( pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -5052,17 +5056,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ + ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); - if( pPager->tempFile && pPager->errCode ) { return pPager->errCode; } assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -5148,7 +5152,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); + rc = pager_playback(pPager, !pPager->tempFile); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -5244,7 +5248,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } diff --git a/test/tempfault.test b/test/tempfault.test index da1ae12375..4eae116a72 100644 --- a/test/tempfault.test +++ b/test/tempfault.test @@ -18,9 +18,9 @@ source $testdir/tester.tcl source $testdir/malloc_common.tcl set testprefix tempfault -sqlite3_memdebug_vfs_oom_test 0 +# sqlite3_memdebug_vfs_oom_test 0 -do_faultsim_test 1 -faults oom* -prep { +do_faultsim_test 1 -faults * -prep { sqlite3 db "" db eval { PRAGMA page_size = 1024; @@ -42,14 +42,14 @@ do_faultsim_test 1 -faults oom* -prep { faultsim_integrity_check } -do_faultsim_test 2 -faults oom* -prep { +do_faultsim_test 2 -faults * -prep { sqlite3 db "" db eval { PRAGMA page_size = 1024; PRAGMA cache_size = 10; CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(b, a); - WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100) INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; } } -body { @@ -59,7 +59,27 @@ do_faultsim_test 2 -faults oom* -prep { faultsim_integrity_check db } -do_faultsim_test 3 -faults oom* -prep { +catch { db close } +do_faultsim_test 2.1 -faults * -prep { + if {[info commands db]==""} { + sqlite3 db "" + execsql { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } + } +} -body { + execsql { UPDATE t1 SET a = randomblob(99) } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 3 -faults * -prep { sqlite3 db "" db eval { PRAGMA page_size = 1024; From 2d36f065e2843175057dba4e4e34fa3df6541357 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Apr 2016 17:24:16 +0000 Subject: [PATCH 14/20] Fix a problem with mixing temp-files and mmap-mode. FossilOrigin-Name: c80c5c62b2e2c5e47e0839f8e2d5b6341ca4a249 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 6 +++--- test/temptable2.test | 29 +++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ec3e51504b..c7f8ab468f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\sto\sdo\swith\stemp-file\sdatabases\sand\srecovering\sfrom\sIO\sand\sSQLITE_FULL\serrors. -D 2016-04-23T14:55:28.020 +C Fix\sa\sproblem\swith\smixing\stemp-files\sand\smmap-mode. +D 2016-04-23T17:24:16.091 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -362,7 +362,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 4981dc6154ce111361a7cd35eaa26aadf8914ea3 +F src/pager.c cbc8996b773c191107b771424b529307ffdf19ba F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1111,7 +1111,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test e62783549be26283e1b3725f0de3309411be6c84 +F test/temptable2.test 31485911fb33e72c7737087ba5a2b35acafba55a F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49aec9718d61c9f12aed96f530128666c3b01c81 -R 98471ae1c4ed8f399ba91e3e119d0ef8 +P 3d61da4a76af8c9c2a293df085f3ed5a7bb447df +R 85df4f851dc9e804fc35d28f4dbcf65b U dan -Z 4115a15d29a4b7f5f76b2491e48cc4f1 +Z 50e48f2ab5de65ac7aff6f1c82162e94 diff --git a/manifest.uuid b/manifest.uuid index 7b1996a745..865e4fbe13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d61da4a76af8c9c2a293df085f3ed5a7bb447df \ No newline at end of file +c80c5c62b2e2c5e47e0839f8e2d5b6341ca4a249 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index a749b8a6b3..3f2a131877 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2034,7 +2034,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( pagerFlushOnCommit(pPager) ){ + if( MEMDB || pagerFlushOnCommit(pPager) ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); @@ -5381,7 +5381,7 @@ int sqlite3PagerGet( ); if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ @@ -6219,7 +6219,7 @@ int sqlite3PagerCommitPhaseOne( assert( MEMDB==0 || pPager->tempFile ); assert( isOpen(pPager->fd) || pPager->tempFile ); - if( !isOpen(pPager->fd) ){ + if( 0==pagerFlushOnCommit(pPager) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any ** backup in progress needs to be restarted. */ diff --git a/test/temptable2.test b/test/temptable2.test index c5e53de635..d71c168ebf 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -316,5 +316,34 @@ foreach {tn mode} { } } +#------------------------------------------------------------------------- +# When using mmap mode with a temp file, SQLite must search the cache +# before using a mapped page even when there is no write transaction +# open. For a temp file, the on-disk version may not be up to date. +# +sqlite3 db "" +do_execsql_test 10.0 { + PRAGMA cache_size = 50; + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 2); +} + +do_execsql_test 10.1 { + BEGIN; + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + COMMIT; + INSERT INTO t2 VALUES(3, 4); +} + +do_execsql_test 10.2 { + PRAGMA mmap_size = 512000; + SELECT * FROM t2; + PRAGMA integrity_check; +} {512000 1 2 3 4 ok} + finish_test From 835f22dedade668d99034711a3758ca32752e716 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 19:20:56 +0000 Subject: [PATCH 15/20] Fix some unreachable branches in the pager. FossilOrigin-Name: 3ae44770fdecc40c8097f1de0b504f36585a2232 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 18 ++++++++++-------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 713cf4421c..acd08f3f4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sbug\sfix\sfrom\strunk. -D 2016-04-25T15:03:49.467 +C Fix\ssome\sunreachable\sbranches\sin\sthe\spager. +D 2016-04-25T19:20:56.371 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -364,7 +364,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c cbc8996b773c191107b771424b529307ffdf19ba +F src/pager.c a31af55dd6139c6ece7642667a035028a266d344 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9b8fec60d8e576cd09e1d075a59bfad1c6169d7a ec215f94ac9748c0acd82af0cc9e7a92249462f9 -R e941df834a8d6e2084dd43227adc9b5d +P a905d5e08de3f3c60b667d840b5995911372647d +R 1f1507f9b43194c78dbe59034ff5cd5c U drh -Z 530f6cc77507869285e9eb18ff03544a +Z 8869ed19649f951f40d0b8fe31c4bd6b diff --git a/manifest.uuid b/manifest.uuid index 02d571129d..010c2a556f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a905d5e08de3f3c60b667d840b5995911372647d \ No newline at end of file +3ae44770fdecc40c8097f1de0b504f36585a2232 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3f2a131877..0a185c3e37 100644 --- a/src/pager.c +++ b/src/pager.c @@ -872,6 +872,7 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ + assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -3204,6 +3205,8 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); /* If the number of pages in the database is not available from the @@ -3211,14 +3214,11 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** the database file. If the size of the database file is not an ** integer multiple of the page-size, round up the result. */ - if( nPage==0 ){ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -4964,6 +4964,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ + assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -5448,7 +5449,8 @@ int sqlite3PagerGet( goto pager_acquire_err; } - if( MEMDB || pPager->dbSizefd) ){ + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; From 6c96358601f1da67c58661706c6066dce51214a8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 19:28:34 +0000 Subject: [PATCH 16/20] Remove an obsolete comment from the pager. No code changes. FossilOrigin-Name: 36f97ca8874a03ac8699f44fe0da95b0be507cc7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index acd08f3f4f..ad1b6577cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sunreachable\sbranches\sin\sthe\spager. -D 2016-04-25T19:20:56.371 +C Remove\san\sobsolete\scomment\sfrom\sthe\spager.\s\sNo\scode\schanges. +D 2016-04-25T19:28:34.469 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -364,7 +364,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c a31af55dd6139c6ece7642667a035028a266d344 +F src/pager.c 456ea27c2a467c161b1466b8f4f5307f12314d31 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a905d5e08de3f3c60b667d840b5995911372647d -R 1f1507f9b43194c78dbe59034ff5cd5c +P 3ae44770fdecc40c8097f1de0b504f36585a2232 +R db63125d858bb9f2f1e3791a791b0b06 U drh -Z 8869ed19649f951f40d0b8fe31c4bd6b +Z 7c3cc56899db1d72bed3171f2d6f3ea7 diff --git a/manifest.uuid b/manifest.uuid index 010c2a556f..5ba52e8d51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ae44770fdecc40c8097f1de0b504f36585a2232 \ No newline at end of file +36f97ca8874a03ac8699f44fe0da95b0be507cc7 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 0a185c3e37..6e464ba82f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1820,10 +1820,7 @@ static void pager_unlock(Pager *pPager){ ** trusted. Now that there are no outstanding references to the pager, ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. - ** - ** Exception: There is no way out of the error state for temp files. - ** This is because it is not possible to call pager_reset() on a temp - ** file pager (as this may discard the only copy of some data). */ + */ assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ if( pPager->tempFile==0 ){ From ead01fd27250ce29a44734a050954ac48434607a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Apr 2016 22:39:42 +0000 Subject: [PATCH 17/20] Add an sqlite3FaultSim() all to make it easier to simulate IO errors in a VACUUM commit. FossilOrigin-Name: 8bfde416c51f4e087275aebf652a73985515e810 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ad1b6577cc..07bba5bb6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sobsolete\scomment\sfrom\sthe\spager.\s\sNo\scode\schanges. -D 2016-04-25T19:28:34.469 +C Add\san\ssqlite3FaultSim()\sall\sto\smake\sit\seasier\sto\ssimulate\sIO\serrors\sin\na\sVACUUM\scommit. +D 2016-04-25T22:39:42.651 F Makefile.in a905f3180accdafbd5a534bf26126ee5306d5056 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -364,7 +364,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 2488a2b6456709ad6398df2302d427a980e2695a F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 456ea27c2a467c161b1466b8f4f5307f12314d31 +F src/pager.c a8d30c49c231e9a20d05257613db922532588963 F src/pager.h 329bdf078a4e0a3b35084534d58625d21fd03681 F src/parse.y 10eb2f3fb62341291528c7984498054731f9d31e F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ae44770fdecc40c8097f1de0b504f36585a2232 -R db63125d858bb9f2f1e3791a791b0b06 +P 36f97ca8874a03ac8699f44fe0da95b0be507cc7 +R 54e681c99d31e5095466803c5f1b64d8 U drh -Z 7c3cc56899db1d72bed3171f2d6f3ea7 +Z 6d6cde3ceb3c27e62273721031ebad91 diff --git a/manifest.uuid b/manifest.uuid index 5ba52e8d51..bbdd0e7139 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36f97ca8874a03ac8699f44fe0da95b0be507cc7 \ No newline at end of file +8bfde416c51f4e087275aebf652a73985515e810 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 6e464ba82f..d24337cd50 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6210,6 +6210,9 @@ int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Provide the ability to easily simulate an I/O error during testing */ + if( (rc = sqlite3FaultSim(400))!=SQLITE_OK ) return rc; + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); From 45164826b76087ae15c714b87d35336aee056637 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Apr 2016 17:10:03 +0000 Subject: [PATCH 18/20] Fix an issue in temptable2.test preventing it from working with SQLITE_DEFAULT_AUTOVACUUM=1 builds. FossilOrigin-Name: e790aac02e4b427b4891b514a050699d159b03b1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3224381eed..52a995d5ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\schecksymbols\sfix\sfrom\strunk\sinto\sthis\sbranch. -D 2016-04-26T16:03:58.388 +C Fix\san\sissue\sin\stemptable2.test\spreventing\sit\sfrom\sworking\swith\sSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds. +D 2016-04-26T17:10:03.745 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -1113,7 +1113,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test 31485911fb33e72c7737087ba5a2b35acafba55a +F test/temptable2.test d4e967c355b154e8bc387de961430e94c20f2dca F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8bfde416c51f4e087275aebf652a73985515e810 d819bfbd46dc7f065d2a3ab0ef9f8322d8394e11 -R 0f2ec21167579c0f8c46594d0d9ecc27 +P 04911cee0cd5467ee99744b774d692219e5e1425 +R 849d538a2e2ee898d96d5ba70eff854f U dan -Z 444c73c0b2d802c9a660647c7b166bf1 +Z 496f6fcde8513321efb142fd9a4f11c8 diff --git a/manifest.uuid b/manifest.uuid index 6798dc8b47..021372e495 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04911cee0cd5467ee99744b774d692219e5e1425 \ No newline at end of file +e790aac02e4b427b4891b514a050699d159b03b1 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index d71c168ebf..21e06a3a4f 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -224,6 +224,7 @@ do_execsql_test 7.1 { # reset_db do_execsql_test 8.1 { + PRAGMA auto_vacuum = OFF; CREATE TABLE t2(a, b); CREATE INDEX i2 ON t2(a, b); WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 ) @@ -234,6 +235,7 @@ do_execsql_test 8.1 { do_test 8.2 { sqlite3 tmp "" execsql { + PRAGMA auto_vacuum = OFF; PRAGMA page_size = 8192; CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); From 9bf0136384c90f4f9e7aacb705575617b459792c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Apr 2016 11:24:42 +0000 Subject: [PATCH 19/20] Fix a couple of test script problems on this branch. Both related to the fact that temp database page sizes can no longer be changed by VACUUM or the backup API after the temp db is populated. FossilOrigin-Name: 6bb5aa2be2a6c5ca66a98a9387b2845507a6bdd2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/backup.test | 21 ++++++++++----------- test/pagerfault.test | 30 ++++++++++++++++-------------- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 52a995d5ff..45d2492933 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sin\stemptable2.test\spreventing\sit\sfrom\sworking\swith\sSQLITE_DEFAULT_AUTOVACUUM=1\sbuilds. -D 2016-04-26T17:10:03.745 +C Fix\sa\scouple\sof\stest\sscript\sproblems\son\sthis\sbranch.\sBoth\srelated\sto\sthe\sfact\sthat\stemp\sdatabase\spage\ssizes\scan\sno\slonger\sbe\schanged\sby\sVACUUM\sor\sthe\sbackup\sAPI\safter\sthe\stemp\sdb\sis\spopulated. +D 2016-04-27T11:24:42.387 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -512,7 +512,7 @@ F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d -F test/backup.test b79299a536a4c6d919094786595b95be56d02014 +F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 @@ -968,7 +968,7 @@ F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test ae9ee0db5a30aecda9db8290ce3dd12e5f7bbaa1 +F test/pagerfault.test 42ff797b1e6426c141cc7ee8b7417c9f27427950 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04911cee0cd5467ee99744b774d692219e5e1425 -R 849d538a2e2ee898d96d5ba70eff854f +P e790aac02e4b427b4891b514a050699d159b03b1 +R 167ee4a4f32b4f8204918ef34f7c21ee U dan -Z 496f6fcde8513321efb142fd9a4f11c8 +Z bbfd58a46d92c2867a1001ebb15e237d diff --git a/manifest.uuid b/manifest.uuid index 021372e495..eadf0ef5a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e790aac02e4b427b4891b514a050699d159b03b1 \ No newline at end of file +6bb5aa2be2a6c5ca66a98a9387b2845507a6bdd2 \ No newline at end of file diff --git a/test/backup.test b/test/backup.test index 3b1e1db9e5..829b32452d 100644 --- a/test/backup.test +++ b/test/backup.test @@ -164,7 +164,7 @@ foreach zOpenScript [list { set file_dest temp }] { foreach rows_dest {0 3 10} { -foreach pgsz_dest {512 1024 2048} { +foreach pgsz_dest {512 1024 2048 4096} { foreach nPagePerStep {1 200} { # Open the databases. @@ -176,17 +176,16 @@ foreach nPagePerStep {1 200} { # in-memory destination is only possible if the initial destination # page size is the same as the source page size (in this case 1024 bytes). # - set isMemDest [expr { - $zDestFile eq ":memory:" || $file_dest eq "temp" && $TEMP_STORE>=2 - }] + set isMemDest [expr { $zDestFile eq ":memory:" || $file_dest eq "temp" }] - if { $isMemDest==0 || $pgsz_dest == 1024 } { - if 0 { - puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" - puts -nonewline " (as $db_dest.$file_dest)" - puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" - puts "" - } + if 0 { + puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" + puts -nonewline " (as $db_dest.$file_dest)" + puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" + puts "" + } + + if { $isMemDest==0 || $pgsz_dest==1024 || $rows_dest==0 } { # Set up the content of the source database. execsql { diff --git a/test/pagerfault.test b/test/pagerfault.test index c0f5de69ac..2e70171b20 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -684,22 +684,24 @@ do_faultsim_test pagerfault-14a -prep { # is not possible to change the page-size of an in-memory database. Even # using the backup API. # -if {$TEMP_STORE<2} { - do_faultsim_test pagerfault-14b -prep { - catch { db2 close } - faultsim_restore_and_reopen +# Update: It is no longer possible to change the page size of any temp +# database after it has been created. +# +do_faultsim_test pagerfault-14b -prep { + catch { db2 close } + faultsim_restore_and_reopen sqlite3 db2 "" db2 eval { PRAGMA page_size = 4096; CREATE TABLE xx(a) } - } -body { - sqlite3_backup B db2 main db main - B step 200 - set rc [B finish] - if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR} - if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] } - set {} {} - } -test { - faultsim_test_result {0 {}} {1 {sqlite3_backup_init() failed}} - } +} -body { + sqlite3_backup B db2 main db main + B step 200 + set rc [B finish] + if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR} + if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] } + set {} {} +} -test { + faultsim_test_result {1 {attempt to write a readonly database}} \ + {1 {sqlite3_backup_init() failed}} } do_faultsim_test pagerfault-14c -prep { From 4a030c643070a939ef2b3f86b56b64a38693868d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 29 Apr 2016 14:12:48 +0000 Subject: [PATCH 20/20] Fix test script temptable2.test so that it works with the "inmemory_journal" and "journaltest" permutations. FossilOrigin-Name: b7bec7f2d3d08d0c14f77654be2c5cedf1619e58 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/temptable2.test | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c04f4869ed..1a928a99b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges,\sincluding\stest\scase\sfixes,\swith\sthis\sbranch. -D 2016-04-29T11:35:28.485 +C Fix\stest\sscript\stemptable2.test\sso\sthat\sit\sworks\swith\sthe\s"inmemory_journal"\sand\s"journaltest"\spermutations. +D 2016-04-29T14:12:48.901 F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -1113,7 +1113,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test d4e967c355b154e8bc387de961430e94c20f2dca +F test/temptable2.test c3d8c138f493207612960bbd6a8c50e84975e2ee F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 F test/tester.tcl 30c7a9be8601d1c1c9c93d013545ebcb28d64254 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 @@ -1486,7 +1486,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6bb5aa2be2a6c5ca66a98a9387b2845507a6bdd2 4cbd50245440e714935399a5c285cf32d8bfc424 -R 7ae3c477f36afa854d04f85029639199 +P 99794aca7b6cb40f08ce4db9889a989fc597eac9 +R f9ef4e1af8653e902d9192ffb04cabc2 U dan -Z 1ae9b89d9de363b516966d4d50de5d49 +Z 5d059cb204727917e4a7564cbd5ed599 diff --git a/manifest.uuid b/manifest.uuid index e7c3b7b953..db83d58f1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99794aca7b6cb40f08ce4db9889a989fc597eac9 \ No newline at end of file +b7bec7f2d3d08d0c14f77654be2c5cedf1619e58 \ No newline at end of file diff --git a/test/temptable2.test b/test/temptable2.test index 21e06a3a4f..890b3b996d 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -139,9 +139,11 @@ do_execsql_test 5.1.1 { INSERT INTO t1 VALUES(1, 2); } +# Test that the temp database is now much bigger than the configured +# cache size (10 pages). do_test 5.1.2 { set n [db one { PRAGMA temp.page_count }] - expr ($n > 280 && $n < 290) + expr ($n > 270 && $n < 290) } {1} do_execsql_test 5.1.3 { @@ -284,9 +286,9 @@ foreach {tn mode} { PRAGMA cache_size = 15; PRAGMA auto_vacuum = 1; } - do_execsql_test 9.$tn.1.2 "PRAGMA journal_mode = $mode" delete + execsql "PRAGMA journal_mode = $mode" - do_execsql_test 9.$tn.1.3 { + do_execsql_test 9.$tn.1.2 { CREATE TABLE tx(a, b); CREATE INDEX i1 ON tx(a); CREATE INDEX i2 ON tx(b); @@ -341,11 +343,14 @@ do_execsql_test 10.1 { INSERT INTO t2 VALUES(3, 4); } -do_execsql_test 10.2 { - PRAGMA mmap_size = 512000; - SELECT * FROM t2; - PRAGMA integrity_check; -} {512000 1 2 3 4 ok} +if {[permutation]!="journaltest"} { + # The journaltest permutation does not support mmap, so this part of + # the test is omitted. + do_execsql_test 10.2 { PRAGMA mmap_size = 512000 } 512000 +} + +do_execsql_test 10.3 { SELECT * FROM t2 } {1 2 3 4} +do_execsql_test 10.4 { PRAGMA integrity_check } ok finish_test