mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allow the database file to be memory mapped in wal mode.

Browse Source 
FossilOrigin-Name: d190ddabc386bc9654b99e33fb81b2f6e67b54d6
This commit is contained in:
dan 2013-03-22 18:20:14 +00:00
parent c00033125d
commit 99bd10979a
6 changed files with 124 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
manifest
View File

@ -1,5 +1,5 @@
C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit.
D 2013-03-22T17:46:11.890
C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode.
D 2013-03-22T18:20:14.143
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -162,8 +162,8 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b
F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf
F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6
F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36
F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c
F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c
F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7
F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95
F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222
@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab
F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9
F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835
F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2
F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6
F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f
F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781
F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b
F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a
F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
P 765615f9fba7c1765eb741cb98a09a28b464ee55
R 125e508e18383c5f0a5b411a39a78cab
P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b
R 7243a86b7c03fc5d49450319f9d9b1bf
U dan
Z 4d1b2a2cebcc3fe5bdd2b5af2205540b
Z 7b1a30ba76756800b3f32db2afc843bf

2
manifest.uuid
View File

@ -1 +1 @@
19345416ed5e1ab5b0b35993b0b9069c2fb1683b
d190ddabc386bc9654b99e33fb81b2f6e67b54d6

88
src/pager.c
View File

@ -2095,7 +2095,7 @@ static void pagerReportSize(Pager *pPager){
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){
int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){
int rc = SQLITE_OK;
if( pPager->nMapValid>=(iOff+nBuf) ){
memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf);
@ -2279,7 +2279,7 @@ static int pager_playback_one_page(
i64 ofst = (pgno-1)*(i64)pPager->pageSize;
testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 );
assert( !pagerUseWal(pPager) );
rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst);
rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst);
if( pgno>pPager->dbFileSize ){
pPager->dbFileSize = pgno;
}
@ -2864,11 +2864,10 @@ end_playback:
** If an IO error occurs, then the IO error is returned to the caller.
** Otherwise, SQLITE_OK is returned.
*/
static int readDbPage(PgHdr *pPg){
static int readDbPage(PgHdr *pPg, u32 iFrame){
Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
Pgno pgno = pPg->pgno; /* Page number to read */
int rc = SQLITE_OK; /* Return code */
int isInWal = 0; /* True if page is in log file */
int pgsz = pPager->pageSize; /* Number of bytes to read */
assert( pPager->eState>=PAGER_READER && !MEMDB );
@ -2880,11 +2879,10 @@ static int readDbPage(PgHdr *pPg){
return SQLITE_OK;
}
if( pagerUseWal(pPager) ){
if( iFrame ){
/* Try to pull the page from the write-ahead log. */
rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData);
}
if( rc==SQLITE_OK && !isInWal ){
rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData);
}else{
i64 iOffset = (pgno-1)*(i64)pPager->pageSize;
if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){
memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize);
@ -2972,7 +2970,13 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){
if( sqlite3PcachePageRefcount(pPg)==1 ){
sqlite3PcacheDrop(pPg);
}else{
rc = readDbPage(pPg);
u32 iFrame = 0;
if( pagerUseWal(pPager) ){
rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame);
}
if( rc==SQLITE_OK ){
rc = readDbPage(pPg, iFrame);
}
if( rc==SQLITE_OK ){
pPager->xReiniter(pPg);
}
@ -3388,7 +3392,6 @@ static void pagerFixMaplimit(Pager *pPager){
|| pPager->fd->pMethods->iVersion<3
|| pPager->fd->pMethods->xMremap==0
|| pPager->tempFile
|| pPager->pWal
){
pPager->nMapLimit = 0;
}else if( pPager->nMapCfgLimit<0 ){
@ -3888,9 +3891,9 @@ static int pagerMap(Pager *pPager, int bExtend){
Pgno nPg; /* Size of mapping to request in pages */
i64 sz; /* Size of mapping to request in bytes */
assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 );
assert( isOpen(pPager->fd) && pPager->tempFile==0 );
assert( pPager->pMap==0 || pPager->nMap>0 );
assert( pPager->eState>=1 );
/* assert( pPager->eState>=1 ); */
assert( pPager->nMmapOut==0 );
assert( pPager->nMapLimit>0 );
@ -4213,6 +4216,46 @@ static int syncJournal(Pager *pPager, int newHdr){
return SQLITE_OK;
}
/*
** This is called by the wal.c module at the start of a checkpoint. If the
** checkpoint runs to completion, it will set the database file size to
** szReq bytes. This function performs two tasks:
**
** * If the file is currently less than szReq bytes in size, an
** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS
** layer of the expected file size, and
**
** * If mmap is being used, then the mapping is extended to szReq
** bytes in size.
**
** SQLITE_OK is returned if successful, or an error code if an error occurs.
*/
int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){
int rc;
sqlite3_int64 sz;
assert( pPager->eState==PAGER_OPEN );
assert( pPager->nMmapOut==0 );
rc = sqlite3OsFileSize(pPager->fd, &sz);
if( rc==SQLITE_OK ){
if( sz>szReq ){
sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz);
}
}
if( rc==SQLITE_OK
&& pPager->nMapLimit>0
&& pPager->nMapValid<szReq
&& pPager->nMapValid<pPager->nMapLimit
){
pPager->dbFileSize = (szReq / pPager->pageSize);
rc = pagerMap(pPager, 1);
}
return rc;
}
/*
** The argument is the first in a linked list of dirty pages connected
** by the PgHdr.pDirty pointer. This function writes each one of the
@ -4275,7 +4318,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){
if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){
pPager->dbFileSize = pPager->dbSize;
pagerMap(pPager, 1);
rc = pagerMap(pPager, 1);
}
}
@ -4301,7 +4344,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){
CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData);
/* Write out the page data. */
pagerWriteData(pPager, pData, pPager->pageSize, offset);
rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset);
/* If page 1 was just written, update Pager.dbFileVers to match
** the value now stored in the database file. If writing this
@ -5275,6 +5318,7 @@ int sqlite3PagerAcquire(
){
int rc = SQLITE_OK;
PgHdr *pPg = 0;
u32 iFrame = 0; /* Frame to read from WAL file */
const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT);
/* It is acceptable to use a read-only (mmap) page for any page except
@ -5287,6 +5331,7 @@ int sqlite3PagerAcquire(
assert( pPager->eState>=PAGER_READER );
assert( assert_pager_state(pPager) );
assert( noContent==0 || bMmapOk==0 );
if( pgno==0 ){
return SQLITE_CORRUPT_BKPT;
@ -5298,7 +5343,12 @@ int sqlite3PagerAcquire(
rc = pPager->errCode;
}else{
if( bMmapOk ){
if( bMmapOk && pagerUseWal(pPager) ){
rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame);
if( rc!=SQLITE_OK ) goto pager_acquire_err;
}
if( iFrame==0 && bMmapOk ){
if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){
rc = pagerMap(pPager, 0);
}
@ -5378,9 +5428,13 @@ int sqlite3PagerAcquire(
memset(pPg->pData, 0, pPager->pageSize);
IOTRACE(("ZERO %p %d\n", pPager, pgno));
}else{
if( pagerUseWal(pPager) && bMmapOk==0 ){
rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame);
if( rc!=SQLITE_OK ) goto pager_acquire_err;
}
assert( pPg->pPager==pPager );
pPager->aStat[PAGER_STAT_MISS]++;
rc = readDbPage(pPg);
rc = readDbPage(pPg, iFrame);
if( rc!=SQLITE_OK ){
goto pager_acquire_err;
}
@ -7063,7 +7117,7 @@ static int pagerOpenWal(Pager *pPager){
** (e.g. due to malloc() failure), return an error code.
*/
if( rc==SQLITE_OK ){
rc = sqlite3WalOpen(pPager->pVfs,
rc = sqlite3WalOpen(pPager->pVfs, pPager,
pPager->fd, pPager->zWal, pPager->exclusiveMode,
pPager->journalSizeLimit, &pPager->pWal
);

4
src/pager.h
View File

@ -174,6 +174,10 @@ int sqlite3SectorSize(sqlite3_file *);
/* Functions used to truncate the database file. */
void sqlite3PagerTruncateImage(Pager*,Pgno);
int sqlite3PagerSetFilesize(Pager *, i64);
/* Write data to the database file */
int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff);
#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL)
void *sqlite3PagerCodec(DbPage *);

67
src/wal.c
View File

@ -412,6 +412,7 @@ struct Wal {
sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */
sqlite3_file *pDbFd; /* File handle for the database file */
sqlite3_file *pWalFd; /* File handle for WAL file */
Pager *pPager; /* Pager object */
u32 iCallback; /* Value to pass to log callback (or 0) */
i64 mxWalSize; /* Truncate WAL to this size upon reset */
int nWiData; /* Size of array apWiData */
@ -1251,6 +1252,7 @@ static void walIndexClose(Wal *pWal, int isDelete){
*/
int sqlite3WalOpen(
sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */
Pager *pPager, /* Pager object handle */
sqlite3_file *pDbFd, /* The open database file */
const char *zWalName, /* Name of the WAL file */
int bNoShm, /* True to run in heap-memory mode */
@ -1291,6 +1293,7 @@ int sqlite3WalOpen(
pRet->zWalName = zWalName;
pRet->syncHeader = 1;
pRet->padToSectorBoundary = 1;
pRet->pPager = pPager;
pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
/* Open file handle on the write-ahead log file. */
@ -1727,10 +1730,7 @@ static int walCheckpoint(
*/
if( rc==SQLITE_OK ){
i64 nReq = ((i64)mxPage * szPage);
rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
if( rc==SQLITE_OK && nSize<nReq ){
sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
}
rc = sqlite3PagerSetFilesize(pWal->pPager, nReq);
}
/* Iterate through the contents of the WAL, copying data to the db file. */
@ -1744,7 +1744,7 @@ static int walCheckpoint(
if( rc!=SQLITE_OK ) break;
iOffset = (iDbpage-1)*(i64)szPage;
testcase( IS_BIG_INT(iOffset) );
rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset);
if( rc!=SQLITE_OK ) break;
}
@ -2287,19 +2287,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){
}
/*
** Read a page from the WAL, if it is present in the WAL and if the
** current read transaction is configured to use the WAL.
** Search the wal file for page pgno. If found, set *piRead to the frame that
** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
** to zero.
**
** The *pInWal is set to 1 if the requested page is in the WAL and
** has been loaded. Or *pInWal is set to 0 if the page was not in
** the WAL and needs to be read out of the database.
** Return SQLITE_OK if successful, or an error code if an error occurs. If an
** error does occur, the final value of *piRead is undefined.
*/
int sqlite3WalRead(
int sqlite3WalFindFrame(
Wal *pWal, /* WAL handle */
Pgno pgno, /* Database page number to read data for */
int *pInWal, /* OUT: True if data is read from WAL */
int nOut, /* Size of buffer pOut in bytes */
u8 *pOut /* Buffer to write page data to */
u32 *piRead /* OUT: Frame number (or zero) */
){
u32 iRead = 0; /* If !=0, WAL frame to return data from */
u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */
@ -2315,7 +2313,7 @@ int sqlite3WalRead(
** WAL were empty.
*/
if( iLast==0 || pWal->readLock==0 ){
*pInWal = 0;
*piRead = 0;
return SQLITE_OK;
}
@ -2386,26 +2384,31 @@ int sqlite3WalRead(
}
#endif
/* If iRead is non-zero, then it is the log frame number that contains the
** required page. Read and return data from the log file.
*/
if( iRead ){
int sz;
i64 iOffset;
sz = pWal->hdr.szPage;
sz = (sz&0xfe00) + ((sz&0x0001)<<16);
testcase( sz<=32768 );
testcase( sz>=65536 );
iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE;
*pInWal = 1;
/* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */
return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
}
*pInWal = 0;
*piRead = iRead;
return SQLITE_OK;
}
/*
** Read the contents of frame iRead from the wal file into buffer pOut
** (which is nOut bytes in size). Return SQLITE_OK if successful, or an
** error code otherwise.
*/
int sqlite3WalReadFrame(
Wal *pWal, /* WAL handle */
u32 iRead, /* Frame to read */
int nOut, /* Size of buffer pOut in bytes */
u8 *pOut /* Buffer to write page data to */
){
int sz;
i64 iOffset;
sz = pWal->hdr.szPage;
sz = (sz&0xfe00) + ((sz&0x0001)<<16);
testcase( sz<=32768 );
testcase( sz>=65536 );
iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE;
/* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */
return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
}
/*
** Return the size of the database in pages (or zero, if unknown).

7
src/wal.h
View File

@ -31,7 +31,6 @@
# define sqlite3WalClose(w,x,y,z) 0
# define sqlite3WalBeginReadTransaction(y,z) 0
# define sqlite3WalEndReadTransaction(z)
# define sqlite3WalRead(v,w,x,y,z) 0
# define sqlite3WalDbsize(y) 0
# define sqlite3WalBeginWriteTransaction(y) 0
# define sqlite3WalEndWriteTransaction(x) 0
@ -54,7 +53,8 @@
typedef struct Wal Wal;
/* Open and close a connection to a write-ahead log. */
int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**);
int sqlite3WalOpen(
sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**);
int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *);
/* Set the limiting size of a WAL file. */
@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *);
void sqlite3WalEndReadTransaction(Wal *pWal);
/* Read a page from the write-ahead log, if it is present. */
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut);
int sqlite3WalFindFrame(Wal *, Pgno, u32 *);
int sqlite3WalReadFrame(Wal *, u32, int, u8 *);
/* If the WAL is not empty, return the size of the database. */
Pgno sqlite3WalDbsize(Wal *pWal);