mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Create a version of the log checksummer that works on big-endian platforms. Remove the 512KB size limit on the log-summary.

Browse Source 
FossilOrigin-Name: 5d6d4423d1def39bd2424703120aa985085c3f8e
This commit is contained in:
dan 2010-04-22 19:14:13 +00:00
parent ede6eb8d1d
commit ce4f05f374
5 changed files with 124 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
manifest
View File

@ -1,5 +1,5 @@
C Further\stests\sand\schanges\srelated\sto\sswitching\sbetween\sWAL\sand\srollback\smodes.
D 2010-04-22T06:27:05
C Create\sa\sversion\sof\sthe\slog\schecksummer\sthat\sworks\son\sbig-endian\splatforms.\sRemove\sthe\s512KB\ssize\slimit\son\sthe\slog-summary.
D 2010-04-22T19:14:14
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c
F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581
F src/log.c edafa806982a281205e71c439e99766e924de938
F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6
F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb
F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e
F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a
@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30
F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f
F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1
F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053
F src/pager.c 06fb9b8b7c5283c62f5258e098d6d1649dde7635
F src/pager.c d83d2ea6d2025554e079f891854b133924305e19
F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444
F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e
F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf
@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61
F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
F test/wal.test fd0e5914599b205172dae33c765368a75feb2462
F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49
F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1
F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47
F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df
@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P 9f4f933f2c6596064fcfc7fb5add87e8c5b57448
R cdccc9135123b3edc2fcb17c83fbd857
P 1236318477787a612f02cc98caf2931bd2e99d94
R 2eba2ad72744173c34347863efb93bd2
U dan
Z 891f9ffc6892beb06734f78c0f8e5711
Z b18c4f1fb26d4b385c95cfa4edf72422

2
manifest.uuid
View File

@ -1 +1 @@
1236318477787a612f02cc98caf2931bd2e99d94
5d6d4423d1def39bd2424703120aa985085c3f8e

118
src/log.c
View File

@ -83,6 +83,15 @@ struct LogSummaryHdr {
LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \
)
/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over
** 13000 frames, so the mapping size does not need to be increased often.
*/
#define LOGSUMMARY_MMAP_INCREMENT (64*1024)
/*
** There is one instance of this structure for each log-summary object
** that this process has a connection to. They are stored in a linked
@ -99,6 +108,7 @@ struct LogSummary {
char *zPath; /* Path to associated WAL file */
LogLock *pLock; /* Linked list of locks on this object */
LogSummary *pNext; /* Next in global list */
int nData; /* Size of aData allocation/mapping */
u32 *aData; /* File body */
};
@ -288,10 +298,22 @@ static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){
assert( LOG_CKSM_BYTES==2*sizeof(u32) );
assert( (nByte&0x00000003)==0 );
do {
sum1 += (*a32++);
sum2 += sum1;
} while( a32<aEnd );
if( SQLITE_LITTLEENDIAN ){
#ifdef SQLITE_DEBUG
u8 *a = (u8 *)a32;
assert( *a32==(a[0] + (a[1]<<8) + (a[2]<<16) + (a[3]<<24)) );
#endif
do {
sum1 += *a32;
sum2 += sum1;
} while( ++a32<aEnd );
}else{
do {
u8 *a = (u8*)a32;
sum1 += a[0] + (a[1]<<8) + (a[2]<<16) + (a[3]<<24);
sum2 += sum1;
} while( ++a32<aEnd );
}
aCksum[0] = sum1 + (sum1>>24);
aCksum[1] = sum2 + (sum2>>24);
@ -333,42 +355,6 @@ static void logNormalizePath(char *zPath){
z[j] = 0;
}
/*
** Memory map the first nByte bytes of the summary file opened with
** pSummary->fd at pSummary->aData. If the summary file is smaller than
** nByte bytes in size when this function is called, ftruncate() is
** used to expand it before it is mapped.
**
** It is assumed that an exclusive lock is held on the summary file
** by the caller (to protect the ftruncate()).
*/
static int logSummaryMap(LogSummary *pSummary, int nByte){
struct stat sStat;
int rc;
int fd = pSummary->fd;
void *pMap;
assert( pSummary->aData==0 );
/* If the file is less than nByte bytes in size, cause it to grow. */
rc = fstat(fd, &sStat);
if( rc!=0 ) return SQLITE_IOERR;
if( sStat.st_size<nByte ){
rc = ftruncate(fd, nByte);
if( rc!=0 ) return SQLITE_IOERR;
}
/* Map the file. */
pMap = mmap(0, nByte, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
if( pMap==MAP_FAILED ){
return SQLITE_IOERR;
}
pSummary->aData = (u32 *)pMap;
pSummary->nData = nByte;
return SQLITE_OK;
}
/*
** Unmap the log-summary mapping and close the file-descriptor. If
** the isTruncate argument is non-zero, truncate the log-summary file
@ -523,6 +509,44 @@ static void logMergesort8(
}
/*
** Memory map the first nByte bytes of the summary file opened with
** pSummary->fd at pSummary->aData. If the summary file is smaller than
** nByte bytes in size when this function is called, ftruncate() is
** used to expand it before it is mapped.
**
** It is assumed that an exclusive lock is held on the summary file
** by the caller (to protect the ftruncate()).
*/
static int logSummaryMap(LogSummary *pSummary, int nByte){
struct stat sStat;
int rc;
int fd = pSummary->fd;
void *pMap;
assert( pSummary->aData==0 );
/* If the file is less than nByte bytes in size, cause it to grow. */
rc = fstat(fd, &sStat);
if( rc!=0 ) return SQLITE_IOERR;
if( sStat.st_size<nByte ){
rc = ftruncate(fd, nByte);
if( rc!=0 ) return SQLITE_IOERR;
}else{
nByte = sStat.st_size;
}
/* Map the file. */
pMap = mmap(0, nByte, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
if( pMap==MAP_FAILED ){
return SQLITE_IOERR;
}
pSummary->aData = (u32 *)pMap;
pSummary->nData = nByte/4;
return SQLITE_OK;
}
/*
** Return the index in the LogSummary.aData array that corresponds to
** frame iFrame. The log-summary file consists of a header, followed by
@ -543,6 +567,16 @@ static int logSummaryEntry(u32 iFrame){
static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
u32 iSlot = logSummaryEntry(iFrame);
if( (iSlot+128)>=pSummary->nData ){
int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;
sqlite3_mutex_enter(pSummary->mutex);
munmap(pSummary->aData, pSummary->nData*4);
pSummary->aData = 0;
logSummaryMap(pSummary, nByte);
sqlite3_mutex_leave(pSummary->mutex);
}
/* Set the log-summary entry itself */
pSummary->aData[iSlot] = iPage;
@ -606,7 +640,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){
** SQLITE_MAX_PAGE_SIZE, conclude that the log file contains no valid data.
*/
nPgsz = sqlite3Get4byte(&aBuf[0]);
if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE ){
if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE || nPgsz<512 ){
goto finished;
}
aCksum[0] = sqlite3Get4byte(&aBuf[4]);
@ -885,7 +919,7 @@ static int logSummaryInit(
assert( sqlite3_mutex_held(pSummary->mutex) );
rc = logLockMutex(pSummary, LOG_WRLOCKW);
if( rc!=SQLITE_OK ) return rc;
rc = logSummaryMap(pSummary, 512*1024);
rc = logSummaryMap(pSummary, LOGSUMMARY_MMAP_INCREMENT);
if( rc!=SQLITE_OK ) goto out;
/* Try to obtain an EXCLUSIVE lock on the dead-mans-hand region. If this

8
src/pager.c
View File

@ -3755,8 +3755,8 @@ static int pagerHasWAL(Pager *pPager, int *pExists){
}
static int pagerOpenSnapshot(Pager *pPager){
int rc;
int changed;
int rc; /* Return code */
int changed = 0; /* True if cache must be reset */
assert( pagerUseLog(pPager) );
@ -3764,8 +3764,8 @@ static int pagerOpenSnapshot(Pager *pPager){
if( rc==SQLITE_OK ){
int dummy;
if( changed ){
pager_reset(pPager);
assert( pPager->errCode || pPager->dbSizeValid==0 );
pager_reset(pPager);
assert( pPager->errCode || pPager->dbSizeValid==0 );
}
rc = sqlite3PagerPagecount(pPager, &dummy);
}

39
test/wal.test
View File

@ -676,7 +676,6 @@ do_test wal-12.3 {
execsql { INSERT INTO t2 VALUES('B', 1) }
list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044]
} {3 2}
do_test wal-12.4 {
file copy -force test.db test2.db
file copy -force test.db-wal test2.db-wal
@ -684,7 +683,6 @@ do_test wal-12.4 {
execsql { SELECT * FROM t2 } db2
} {B 1}
db2 close
do_test wal-12.5 {
execsql {
PRAGMA checkpoint;
@ -696,14 +694,47 @@ do_test wal-12.5 {
SELECT * FROM t2;
}
} {B 2}
do_test wal-12.4 {
do_test wal-12.6 {
file copy -force test.db test2.db
file copy -force test.db-wal test2.db-wal
sqlite3_wal db2 test2.db
execsql { SELECT * FROM t2 } db2
} {B 2}
db2 close
db close
#-------------------------------------------------------------------------
# Test large log summaries.
#
do_test wal-13.1 {
list [file exists test.db] [file exists test.db-wal]
} {1 0}
do_test wal-13.2 {
set fd [open test.db-wal w]
seek $fd [expr 200*1024*1024]
puts $fd ""
close $fd
sqlite3 db test.db
execsql { SELECT * FROM t2 }
} {B 2}
do_test wal-13.3 {
db close
file exists test.db-wal
} {0}
do_test wal-13.4 {
sqlite3 db test.db
execsql { SELECT count(*) FROM t2 }
} {1}
do_test wal-13.5 {
for {set i 0} {$i < 15} {incr i} {
execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
}
execsql { SELECT count(*) FROM t2 }
} [expr int(pow(2, 15))]
do_test wal-13.6 {
file size test.db-wal-summary
} [expr 192*1024]
finish_test