From b6be675dcbf0a9895a5daefa21c86a6426e2b174 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Wed, 5 Sep 2007 11:34:54 +0000 Subject: [PATCH] Fix a problem in the test scripts for the asynchronous backend. (CVS 4400) FossilOrigin-Name: 630fc71f3df5ab6129ddff9d8184893ecc6cf3c5 --- manifest | 18 +++---- manifest.uuid | 2 +- src/test_async.c | 123 ++++++++++++++++++++++++----------------------- test/async.test | 19 ++++++-- test/async2.test | 7 +-- 5 files changed, 93 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index 2feeac4cb4..6c50a38ad9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sthe\sTryEnterCriticalSection\sAPI\son\swindows\ssince\sit\sis\nunavailable\son\ssome\splatforms.\s(CVS\s4399) -D 2007-09-04T22:31:37 +C Fix\sa\sproblem\sin\sthe\stest\sscripts\sfor\sthe\sasynchronous\sbackend.\s(CVS\s4400) +D 2007-09-05T11:34:54 F Makefile.in cbfb898945536a8f9ea8b897e1586dd1fdbcc5db F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -145,7 +145,7 @@ F src/test6.c 0513982dfef4da2a4154b538d2bf538b84ca21d3 F src/test7.c a9d509d0e9ad214b4772696f49f6e61be26213d1 F src/test8.c f113aa3723a52113d0fa7c28155ecd37e7e04077 F src/test9.c b46c8fe02ac7cca1a7316436d8d38d50c66f4b2f -F src/test_async.c e221db3e87b472733a8015be7d70bae0edb848b1 +F src/test_async.c 5d30feff6238f083eb32a55f5c18b036a1a5e40c F src/test_autoext.c 855157d97aa28cf84233847548bfacda21807436 F src/test_btree.c c1308ba0b88ab577fa56c9e493a09829dfcded9c F src/test_config.c 6fb459214b27952b143f45e35200d94096d54cc6 @@ -180,8 +180,8 @@ F test/alter2.test 50c3f554b8236d179d72511c0a4f23c5eb7f2af3 F test/alter3.test a6eec8f454be9b6ce73d8d7dc711453675a10ce7 F test/altermalloc.test 1f4d2d66750bea1a78cd9f0b7dba5bfb155dd6cf F test/analyze.test 2f55535aa335785db1a2f97d3f3831c16c09f8b0 -F test/async.test 18e7dc66535f3d86c05e0f954384472e2ed52490 -F test/async2.test a8ef7abfda880b171b2f0a8476300816e33a808a +F test/async.test c52216f8bdebff26900a338b75ea6079944bf141 +F test/async2.test 75f2d15f4c27189ec3296cf2565ec91834bbed76 F test/attach.test b849e1baae863c3a6132ff8b9b1baf356ab6c178 F test/attach2.test 78bc1a25ea8785c7571b44f5947ada2bd5d78127 F test/attach3.test eafcafb107585aecc2ed1569a77914138eef46a9 @@ -569,7 +569,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 3794dcd31a74e90b181b336bf6a4c917bda526b8 -R f8c85691f2efa8fda77b1b74d5099eab -U drh -Z 971dbe3f56f98d850a595d781e5442d1 +P bf3d67d1bd1c48fff45dc24818b8358f79c9fdef +R 25fb668c32b81804b4ade4316a54624b +U danielk1977 +Z 8a09ee32bd86264ddd0da49c7bb5aaae diff --git a/manifest.uuid b/manifest.uuid index 96875eff4a..b4e2b0cc57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf3d67d1bd1c48fff45dc24818b8358f79c9fdef \ No newline at end of file +630fc71f3df5ab6129ddff9d8184893ecc6cf3c5 \ No newline at end of file diff --git a/src/test_async.c b/src/test_async.c index e03eda2c20..13138cfb35 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -24,8 +24,8 @@ ** Asynchronous I/O appears to give better responsiveness, but at a price. ** You lose the Durable property. With the default I/O backend of SQLite, ** once a write completes, you know that the information you wrote is -** safely on disk. With the asynchronous I/O, this is no the case. If -** your program crashes or if you take a power lose after the database +** safely on disk. With the asynchronous I/O, this is not the case. If +** your program crashes or if a power lose occurs after the database ** write but before the asynchronous write thread has completed, then the ** database change might never make it to disk and the next user of the ** database might not see your change. @@ -36,26 +36,14 @@ ** ** HOW IT WORKS ** -** Asynchronous I/O works by overloading the OS-layer disk I/O routines -** with modified versions that store the data to be written in queue of -** pending write operations. Look at the asyncEnable() subroutine to see -** how overloading works. Six os-layer routines are overloaded: +** Asynchronous I/O works by creating a special SQLite "vfs" structure +** and registering it with sqlite3_vfs_register(). When files opened via +** this vfs are written to (using sqlite3OsWrite()), the data is not +** written directly to disk, but is placed in the "write-queue" to be +** handled by the background thread. ** -** sqlite3OsOpenReadWrite; -** sqlite3OsOpenReadOnly; -** sqlite3OsOpenExclusive; -** sqlite3OsDelete; -** sqlite3OsFileExists; -** sqlite3OsSyncDirectory; -** -** The original implementations of these routines are saved and are -** used by the writer thread to do the real I/O. The substitute -** implementations typically put the I/O operation on a queue -** to be handled later by the writer thread, though read operations -** must be handled right away, obviously. -** -** Asynchronous I/O is disabled by setting the os-layer interface routines -** back to their original values. +** The special vfs is registered (and unregistered) by calls to +** function asyncEnable() (see below). ** ** LIMITATIONS ** @@ -73,7 +61,10 @@ /* ** If this symbol is defined, then file-system locks are obtained as ** required. This slows things down, but allows multiple processes -** to access the database concurrently. +** to access the database concurrently. If this symbol is not defined, +** then connections from within a single process will respect each +** others database locks, but external connections will not - leading +** to database corruption. */ #define ENABLE_FILE_LOCKING @@ -125,7 +116,11 @@ static void asyncTrace(const char *zFormat, ...){ ** Basic rules: ** ** * Both read and write access to the global write-op queue must be -** protected by the async.queueMutex. +** protected by the async.queueMutex. As are the async.ioError and +** async.nFile variables. +** +** * The async.aLock hash-table and all AsyncLock and AsyncFileLock +** structures must be protected by teh async.lockMutex mutex. ** ** * The file handles from the underlying system are assumed not to ** be thread safe. @@ -308,17 +303,35 @@ struct AsyncWrite { AsyncWrite *pNext; /* Next write operation (to any file) */ }; +/* +** An instance of this structure is created for each distinct open file +** (i.e. if two handles are opened on the one file, only one of these +** structures is allocated) and stored in the async.aLock hash table. The +** keys for async.aLock are the full pathnames of the opened files. +** +** AsyncLock.pList points to the head of a linked list of AsyncFileLock +** structures, one for each handle currently open on the file. +** +** If the opened file is not a main-database (the SQLITE_OPEN_MAIN_DB is +** not passed to the sqlite3OsOpen() call), or if ENABLE_FILE_LOCKING is +** not defined at compile time, variables AsyncLock.pFile and +** AsyncLock.eLock are never used. Otherwise, pFile is a file handle +** opened on the file in question and used to obtain the file-system +** locks required by database connections within this process. +** +** See comments above the asyncLock() function for more details on +** the implementation of database locking used by this backend. +*/ +struct AsyncLock { + sqlite3_file *pFile; + int eLock; + AsyncFileLock *pList; +}; + /* ** An instance of the following structure is allocated along with each ** AsyncFileData structure (see AsyncFileData.lock), but is only used if the ** file was opened with the SQLITE_OPEN_MAIN_DB. -** -** The global async.aLock[] hash table maps from database file-name to a -** linked-list of AsyncFileLock structures corresponding to handles opened on -** the file. The AsyncFileLock structures are linked into the list when the -** file is opened and removed when it is closed. Mutex async.lockMutex must be -** held before accessing any AsyncFileLock structure or the async.aLock[] -** table. */ struct AsyncFileLock { int eLock; /* Internally visible lock state (sqlite pov) */ @@ -326,12 +339,6 @@ struct AsyncFileLock { AsyncFileLock *pNext; }; -struct AsyncLock { - sqlite3_file *pFile; - int eLock; - AsyncFileLock *pList; -}; - /* ** The AsyncFile structure is a subclass of sqlite3_file used for ** asynchronous IO. @@ -611,6 +618,7 @@ static int getFileLock(AsyncLock *pLock){ assert(pIter->eAsyncLock>=pIter->eLock); if( pIter->eAsyncLock>eRequired ){ eRequired = pIter->eAsyncLock; + assert(eRequired>=0 && eRequired<=SQLITE_LOCK_EXCLUSIVE); } } if( eRequired>pLock->eLock ){ @@ -627,12 +635,9 @@ static int getFileLock(AsyncLock *pLock){ } /* -** No disk locking is performed. We keep track of locks locally in -** the async.aLock hash table. Locking should appear to work the same -** as with standard (unmodified) SQLite as long as all connections -** come from this one process. Connections from external processes -** cannot see our internal hash table (obviously) and will thus not -** honor our locks. +** The following two methods - asyncLock() and asyncUnlock() - are used +** to obtain and release locks on database files opened with the +** asynchronous backend. */ static int asyncLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; @@ -656,9 +661,7 @@ static int asyncLock(sqlite3_file *pFile, int eLock){ } if( rc==SQLITE_OK ){ p->lock.eLock = eLock; - if( eLock>p->lock.eAsyncLock ){ - p->lock.eAsyncLock = eLock; - } + p->lock.eAsyncLock = MAX(p->lock.eAsyncLock, eLock); } assert(p->lock.eAsyncLock>=p->lock.eLock); if( rc==SQLITE_OK ){ @@ -674,9 +677,7 @@ static int asyncUnlock(sqlite3_file *pFile, int eLock){ AsyncFileData *p = ((AsyncFile *)pFile)->pData; AsyncFileLock *pLock = &p->lock; pthread_mutex_lock(&async.lockMutex); - if( pLock->eLock>eLock ){ - pLock->eLock = eLock; - } + pLock->eLock = MIN(pLock->eLock, eLock); pthread_mutex_unlock(&async.lockMutex); return addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0); } @@ -1010,16 +1011,19 @@ static sqlite3_vfs async_vfs = { static void asyncEnable(int enable){ if( enable ){ if( !async_vfs.pAppData ){ + static int hashTableInit = 0; async_vfs.pAppData = (void *)sqlite3_vfs_find(0); async_vfs.mxPathname = ((sqlite3_vfs *)async_vfs.pAppData)->mxPathname; sqlite3_vfs_register(&async_vfs, 1); - sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1); + if( !hashTableInit ){ + sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1); + hashTableInit = 1; + } } }else{ if( async_vfs.pAppData ){ sqlite3_vfs_unregister(&async_vfs); async_vfs.pAppData = 0; - sqlite3HashClear(&async.aLock); } } } @@ -1149,14 +1153,19 @@ static void *asyncWriterThread(void *NotUsed){ pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){ if( (*ppIter)==&pData->lock ){ - *ppIter = (*ppIter)->pNext; + *ppIter = pData->lock.pNext; break; } } if( !pLock->pList ){ - if( pLock->pFile ) sqlite3OsClose(pLock->pFile); + if( pLock->pFile ){ + sqlite3OsClose(pLock->pFile); + } sqlite3_free(pLock); sqlite3HashInsert(&async.aLock, pData->zName, pData->nName, 0); + if( !sqliteHashFirst(&async.aLock) ){ + sqlite3HashClear(&async.aLock); + } }else{ rc = getFileLock(pLock); } @@ -1171,13 +1180,9 @@ static void *asyncWriterThread(void *NotUsed){ AsyncFileData *pData = p->pFileData; int eLock = p->nByte; pthread_mutex_lock(&async.lockMutex); - if( pData->lock.eAsyncLock>eLock ){ - if( pData->lock.eLock>eLock ){ - pData->lock.eAsyncLock = pData->lock.eLock; - }else{ - pData->lock.eAsyncLock = eLock; - } - } + pData->lock.eAsyncLock = MIN( + pData->lock.eAsyncLock, MAX(pData->lock.eLock, eLock) + ); assert(pData->lock.eAsyncLock>=pData->lock.eLock); pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); rc = getFileLock(pLock); diff --git a/test/async.test b/test/async.test index 8575555565..ed2325e344 100644 --- a/test/async.test +++ b/test/async.test @@ -6,7 +6,7 @@ #*********************************************************************** # This file runs all tests. # -# $Id: async.test,v 1.9 2007/09/04 18:28:44 danielk1977 Exp $ +# $Id: async.test,v 1.10 2007/09/05 11:34:54 danielk1977 Exp $ if {[catch {sqlite3async_enable}]} { @@ -18,7 +18,11 @@ if {[catch {sqlite3async_enable}]} { set testdir [file dirname $argv0] source $testdir/tester.tcl rename finish_test really_finish_test -proc finish_test {} {} +proc finish_test {} { + catch {db close} + catch {db2 close} + catch {db3 close} +} set ISQUICK 1 set INCLUDE { @@ -34,7 +38,6 @@ set INCLUDE { lock3.test lock2.test } -# set INCLUDE lock4.test # Enable asynchronous IO. sqlite3async_enable 1 @@ -51,7 +54,15 @@ foreach testfile [lsort -dictionary [glob $testdir/*.test]] { set tail [file tail $testfile] if {[lsearch -exact $INCLUDE $tail]<0} continue source $testfile - catch {db close} + + # Make sure everything is flushed through. This is because [source]ing + # the next test file will delete the database file on disk (using + # [file delete]). If the asynchronous backend still has the file + # open, it will become confused. + # + sqlite3async_halt idle + sqlite3async_start + sqlite3async_wait } # Flush the write-queue and disable asynchronous IO. This should ensure diff --git a/test/async2.test b/test/async2.test index 2d938a657e..3403ea44c0 100644 --- a/test/async2.test +++ b/test/async2.test @@ -5,7 +5,7 @@ # #*********************************************************************** # -# $Id: async2.test,v 1.6 2007/08/30 10:49:55 danielk1977 Exp $ +# $Id: async2.test,v 1.7 2007/09/05 11:34:54 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -70,7 +70,8 @@ foreach err [list ioerr malloc-transient malloc-persistent] { sqlite3async_halt idle sqlite3async_start sqlite3async_wait - + sqlite3async_enable 0 + set ::sqlite_io_error_pending 0 sqlite3_memdebug_fail -1 @@ -113,7 +114,7 @@ foreach err [list ioerr malloc-transient malloc-persistent] { } } - sqlite3async_enable 0 + db close } }