diff --git a/Makefile.in b/Makefile.in index 22d200b10d..eb9389172e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -157,7 +157,7 @@ OBJS0 = alter.lo analyze.lo attach.lo auth.lo bitvec.lo btmutex.lo \ mutex_os2.lo mutex_unix.lo mutex_w32.lo \ opcodes.lo os.lo os_unix.lo os_win.lo os_os2.lo \ pager.lo parse.lo pragma.lo prepare.lo printf.lo random.lo \ - select.lo table.lo tokenize.lo trigger.lo update.lo \ + select.lo status.lo table.lo tokenize.lo trigger.lo update.lo \ util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbefifo.lo vdbemem.lo \ where.lo utf.lo legacy.lo vtab.lo @@ -226,6 +226,7 @@ SRC = \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ $(TOP)/src/select.c \ + $(TOP)/src/status.c \ $(TOP)/src/shell.c \ $(TOP)/src/sqlite.h.in \ $(TOP)/src/sqlite3ext.h \ @@ -610,6 +611,9 @@ random.lo: $(TOP)/src/random.c $(HDR) select.lo: $(TOP)/src/select.c $(HDR) $(LTCOMPILE) -c $(TOP)/src/select.c +status.lo: $(TOP)/src/status.c $(HDR) + $(LTCOMPILE) -c $(TOP)/src/status.c + sqlite3.h: $(TOP)/src/sqlite.h.in sed -e s/--VERS--/$(RELEASE)/ $(TOP)/src/sqlite.h.in | \ sed -e s/--VERSION-NUMBER--/$(VERSION_NUMBER)/ >sqlite3.h diff --git a/main.mk b/main.mk index 0c0f2adcdc..4fb40de298 100644 --- a/main.mk +++ b/main.mk @@ -55,7 +55,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o bitvec.o btmutex.o btree.o build.o \ mutex.o mutex_os2.o mutex_unix.o mutex_w32.o \ opcodes.o os.o os_os2.o os_unix.o os_win.o \ pager.o parse.o pragma.o prepare.o printf.o random.o \ - select.o table.o $(TCLOBJ) tokenize.o trigger.o \ + select.o status.o table.o $(TCLOBJ) tokenize.o trigger.o \ update.o util.o vacuum.o \ vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbefifo.o vdbemem.o \ where.o utf.o legacy.o vtab.o rtree.o @@ -133,6 +133,7 @@ SRC = \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ $(TOP)/src/select.c \ + $(TOP)/src/status.c \ $(TOP)/src/shell.c \ $(TOP)/src/sqlite.h.in \ $(TOP)/src/sqlite3ext.h \ diff --git a/manifest b/manifest index a3d2f0dd97..b3ae349dab 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C fix\sOS/2\sfiles\sto\scompile\sagain\s(looking\sat\sWindows\sequivalents\sfor\sguidance)\s(CVS\s5239) -D 2008-06-18T21:08:16 +C Add\ssome\stest\slogic\sto\sthe\snew\smemory\sallocation\ssubsystem.\s\s(Lots\smore\sneeded.)\nThe\stest\ssuite\sis\scurrently\sindicating\smemory\sleaks,\sthough\sit\sis\sunclear\sif\nthis\sis\sa\strue\scode\sproblem\sor\sjust\san\sinstrumentation\sproblem.\s(CVS\s5240) +D 2008-06-19T00:16:08 F Makefile.arm-wince-mingw32ce-gcc ac5f7b2cef0cd850d6f755ba6ee4ab961b1fadf7 -F Makefile.in dc5608df93faf4406cfd7a1c8ed9ab93d8bfbfd5 +F Makefile.in ff6f90048555a0088f6a4b7406bed5e55a7c4eff F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F README b974cdc3f9f12b87e851b04e75996d720ebf81ac F VERSION d739ba532eddc5c09ef9a55151328cd6a8a102c6 @@ -76,7 +76,7 @@ F ext/rtree/rtree_util.tcl ee0a0311eb12175319d78bfb37302320496cee6e F ext/rtree/viewrtree.tcl 09526398dae87a5a87c5aac2b3854dbaf8376869 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh 09fe5815427dc7d0abb188bbcdf0e34896577210 -F main.mk 8a6820d4ed89c95b4c28c88412bb8bf0513cfc5c +F main.mk f7be21bc9c67879c249b86b332a0b4f8eeef3d3b F mkdll.sh 712e74f3efe08a6ba12b2945d018a29a89d7fe3b F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 1a866b53637dab137191341cc875575a5ca110fb @@ -96,7 +96,7 @@ F src/attach.c b18ba42c77f7d3941f5d23d2ca20fa1d841a4e91 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627 F src/bitvec.c ab50c4b8c6a899dae499f5a805eebe4223c78269 F src/btmutex.c 483ced3c52205b04b97df69161fadbf87f4f1ea2 -F src/btree.c bae7f135dad8fd72ac41ce3c13b7b7ba9fe4d056 +F src/btree.c a7c8e782cf6a36eb4b15f263200215b572f95406 F src/btree.h b1bd7e0b8c2e33658aaf447cb0d1d94f74664b6b F src/btreeInt.h 02325f04758dba0fcd0c08ac55cd9b189dad61a5 F src/build.c 88cc5501a87f72d0538b040001d88d31f994edea @@ -115,8 +115,8 @@ F src/insert.c c2ead6c36566de8e3f130e7ab1431723a269d5d7 F src/journal.c cffd2cd214e58c0e99c3ff632b3bee6c7cbb260e F src/legacy.c 3626c71fb70912abec9a4312beba753a9ce800df F src/loadext.c 2ac671f42a8ce15e2a58155b9d7f6c61eb7e127e -F src/main.c d2f731a9c3ab71ffdc5a34afac968e3b6d2c269c -F src/malloc.c 56e48f8147e04f8f8a19fed8e09be351bdf1a48e +F src/main.c cac2d0736589536c2a11a80676b18806dabff21f +F src/malloc.c 33b8a20684e65ce13f94e5b14fb0e6cdc335c5d1 F src/md5.c 008216bbb5d34c6fbab5357aa68575ad8a31516a F src/mem1.c 159f10e280f2d9aea597cf938851e61652dd5c3d F src/mem2.c 468a7fd5e0ce1909cfb554d7589af1d4a61f14bb @@ -143,9 +143,9 @@ F src/printf.c 8b063da9dcde26b7c500a01444b718d86f21bc6e F src/random.c 362b62e26db90ec9296903377a9d74b4669aa515 F src/select.c 669687459e7d0193c89de06c5dbed55b4a41191c F src/shell.c a12ea645271b7876c8f080146f48e20b00d367ec -F src/sqlite.h.in 85032a736b18b01df995212c5ace20d403245d88 +F src/sqlite.h.in f69427508e76eec76dae7f5fffdfefc05a9d4329 F src/sqlite3ext.h faacd0e6a81aabee0861c6d7883c9172e74ef5b3 -F src/sqliteInt.h af44587156c5e4aa12bf6a6784126735551b4a1d +F src/sqliteInt.h 8c2532dd9f53b05d2d6ec9e772d726718a0d4ac6 F src/sqliteLimit.h f435e728c6b620ef7312814d660a81f9356eb5c8 F src/table.c 1fa8f8113ac9cbc09ae4801c6d2a7f0af82c5822 F src/tclsqlite.c 4dd9ee4cb44846ad9bcc4d0da8088c1e7d4b33d9 @@ -166,7 +166,7 @@ F src/test_devsym.c 6012cb8e3acf812513511025a4fa5d626e0ba19b F src/test_func.c f4aafa10f17d52c43a64b47717265802e6e552b3 F src/test_hexio.c 2f1122aa3f012fa0142ee3c36ce5c902a70cd12f F src/test_loadext.c 22065d601a18878e5542191001f0eaa5d77c0ed8 -F src/test_malloc.c d44d9ee715ad97503c87d287a802b22421a4c32b +F src/test_malloc.c dc6d256544b1be96312367b24b93a778de8afdfe F src/test_md5.c 28209a4e2068711b5443c33104fe41f21d160071 F src/test_mutex.c 08640d7547bfadb6997a22d72e63914feaf4bc4f F src/test_onefile.c 1f87d4a21cbfb24a7c35e4333fa0bd34d641f68d @@ -405,6 +405,7 @@ F test/malloc_common.tcl fd7040bbb0bbbe84187c7f80049fdf6b2a4d699b F test/manydb.test 8de36b8d33aab5ef295b11d9e95310aeded31af8 F test/memdb.test a67bda4ff90a38f2b19f6c7f95aa7289e051d893 F test/memleak.test d2d2a1ff7105d32dc3fdf691458cf6cba58c7217 +F test/memsubsys1.test 353d51691b0fbc10633ab61986d85a6b03cb0533 F test/minmax.test 5d56f08a7765dfb5c1fb303333f7444dacb37bef F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0 F test/minmax3.test 05110398e065875b3d9315892889d3c87fccfe2b @@ -478,7 +479,7 @@ F test/tableapi.test 791f7e3891d9b70bdb43b311694bf5e9befcbc34 F test/tclsqlite.test 3dfb48f46de4353376fad835390b493ba066b4dc F test/tempdb.test b88ac8a19823cf771d742bf61eef93ef337c06b1 F test/temptable.test 19b851b9e3e64d91e9867619b2a3f5fffee6e125 -F test/tester.tcl afb901226b414b60ea136765d0fbf9ffcdf617a1 +F test/tester.tcl 1b57a6bfe913c6b2a12a32e21263bb5dcb0e08e1 F test/thread001.test 8fbd9559da0bbdc273e00318c7fd66c162020af7 F test/thread002.test 2c4ad2c386f60f6fe268cd91c769ee35b3c1fd0b F test/thread1.test 776c9e459b75ba905193b351926ac4019b049f35 @@ -582,7 +583,7 @@ F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8 F tool/memleak3.tcl 7707006ee908cffff210c98158788d85bb3fcdbf F tool/mkkeywordhash.c ef93810fc41fb3d3dbacf9a33a29be88ea99ffa9 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x -F tool/mksqlite3c.tcl 7641043a1c17669a3ecd181558b70c0652e1a9e4 +F tool/mksqlite3c.tcl 5012f63f6366db547fa5a98ec62e9c8fd0e6cd47 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl 7d1fdf469e2f4d175f70c36e469db64a1626fabb F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -597,7 +598,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 1dbced29de5f59ba2ebf877edcadf171540374d1 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e -P 42a2a8f49324e2e07b81fd08e24f636a2d98a961 -R e6e96da06a6e46fe7a401745505f409f -U pweilbacher -Z 5510fdf0cc10dd25a8cc1f913a04d5af +P 8b14a220f261b354e7d2d16dc3fe30c5d3d34143 +R a54a916e07bd9133d83929df10e422cc +U drh +Z 67cf172775ef7e1849c4e693b07767c0 diff --git a/manifest.uuid b/manifest.uuid index 470323e1ba..ca9552a15d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b14a220f261b354e7d2d16dc3fe30c5d3d34143 \ No newline at end of file +cb1f11cd9764cf0275e88e1f6342e366e5536bfd \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 3e3beb3273..9bc560dcde 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.466 2008/06/18 17:09:10 danielk1977 Exp $ +** $Id: btree.c,v 1.467 2008/06/19 00:16:08 drh Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. @@ -1400,6 +1400,24 @@ static int removeFromSharingList(BtShared *pBt){ #endif } +/* +** Make sure pBt->pTmpSpace points to an allocation of +** MX_CELL_SIZE(pBt) bytes. +*/ +static void allocateTempSpace(BtShared *pBt){ + if( !pBt->pTmpSpace ){ + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + } +} + +/* +** Free the pBt->pTmpSpace allocation +*/ +static void freeTempSpace(BtShared *pBt){ + sqlite3PageFree( pBt->pTmpSpace); + pBt->pTmpSpace = 0; +} + /* ** Close an open database and invalidate all cursors. */ @@ -1444,8 +1462,7 @@ int sqlite3BtreeClose(Btree *p){ pBt->xFreeSchema(pBt->pSchema); } sqlite3_free(pBt->pSchema); - sqlite3_free(pBt->pTmpSpace); - sqlite3_free(pBt); + freeTempSpace(pBt); } #ifndef SQLITE_OMIT_SHARED_CACHE @@ -1549,8 +1566,7 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve){ assert( (pageSize & 7)==0 ); assert( !pBt->pPage1 && !pBt->pCursor ); pBt->pageSize = pageSize; - sqlite3_free(pBt->pTmpSpace); - pBt->pTmpSpace = 0; + freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); } pBt->usableSize = pBt->pageSize - nReserve; @@ -1698,8 +1714,7 @@ static int lockBtree(BtShared *pBt){ releasePage(pPage1); pBt->usableSize = usableSize; pBt->pageSize = pageSize; - sqlite3_free(pBt->pTmpSpace); - pBt->pTmpSpace = 0; + freeTempSpace(pBt); sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); return SQLITE_OK; } @@ -5677,16 +5692,6 @@ static int checkReadLocks( return SQLITE_OK; } -/* -** Make sure pBt->pTmpSpace points to an allocation of -** MX_CELL_SIZE(pBt) bytes. -*/ -static void allocateTempSpace(BtShared *pBt){ - if( !pBt->pTmpSpace ){ - pBt->pTmpSpace = sqlite3Malloc(MX_CELL_SIZE(pBt)); - } -} - /* ** Insert a new record into the BTree. The key is given by (pKey,nKey) ** and the data is given by (pData,nData). The cursor is used only to @@ -6643,8 +6648,9 @@ static int checkTreePage( */ data = pPage->aData; hdr = pPage->hdrOffset; - hit = sqlite3MallocZero( usableSize ); + hit = sqlite3PageMalloc( pBt->pageSize ); if( hit ){ + memset(hit, 0, usableSize ); memset(hit, 1, get2byte(&data[hdr+5])); nCell = get2byte(&data[hdr+3]); cellStart = hdr + 12 - 4*pPage->leaf; @@ -6686,7 +6692,7 @@ static int checkTreePage( cnt, data[hdr+7], iPage); } } - sqlite3_free(hit); + sqlite3PageFree(hit); releasePage(pPage); return depth+1; diff --git a/src/main.c b/src/main.c index 433aabdc75..4f61cd2ba3 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,7 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: main.c,v 1.450 2008/06/18 18:57:42 danielk1977 Exp $ +** $Id: main.c,v 1.451 2008/06/19 00:16:08 drh Exp $ */ #include "sqliteInt.h" #include @@ -80,6 +80,7 @@ int sqlite3_initialize(void){ sqlite3_mutex_enter(pMutex); if( sqlite3IsInit==0 ){ sqlite3IsInit = 1; + sqlite3StatusReset(); if( rc==SQLITE_OK ) rc = sqlite3MallocInit(); if( rc==SQLITE_OK ) rc = sqlite3_os_init(); if( rc!=SQLITE_OK ){ diff --git a/src/malloc.c b/src/malloc.c index a278e48166..3441123c69 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -12,7 +12,7 @@ ** ** Memory allocation functions used throughout sqlite. ** -** $Id: malloc.c,v 1.20 2008/06/18 18:12:04 drh Exp $ +** $Id: malloc.c,v 1.21 2008/06/19 00:16:08 drh Exp $ */ #include "sqliteInt.h" #include @@ -97,14 +97,6 @@ static struct { /* Number of free pages for scratch and page-cache memory */ u32 nScratchFree; u32 nPageFree; - - /* - ** Performance statistics - */ - sqlite3_int64 nowUsed; /* Main memory currently in use */ - sqlite3_int64 mxUsed; /* Highwater mark for nowUsed */ - int mxReq; /* Max request size for ordinary mallocs */ - int mxScratchReq; /* Max request size for xTemp mallocs */ } mem0; /* @@ -127,6 +119,7 @@ int sqlite3MallocInit(void){ mem0.nScratchFree = sqlite3Config.nScratch; }else{ sqlite3Config.pScratch = 0; + sqlite3Config.szScratch = 0; } if( sqlite3Config.pPage && sqlite3Config.szPage>=512 && sqlite3Config.nPage>0 ){ @@ -137,6 +130,7 @@ int sqlite3MallocInit(void){ mem0.nPageFree = sqlite3Config.nPage; }else{ sqlite3Config.pPage = 0; + sqlite3Config.szPage = 0; } return sqlite3Config.m.xInit(sqlite3Config.m.pAppData); } @@ -153,12 +147,9 @@ void sqlite3MallocEnd(void){ ** Return the amount of memory currently checked out. */ sqlite3_int64 sqlite3_memory_used(void){ - sqlite3_int64 n; - sqlite3_initialize(); - sqlite3_mutex_enter(mem0.mutex); - n = mem0.nowUsed; - sqlite3_mutex_leave(mem0.mutex); - return n; + int n, mx; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); + return (sqlite3_int64)n; } /* @@ -167,15 +158,9 @@ sqlite3_int64 sqlite3_memory_used(void){ ** or since the most recent reset. */ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ - sqlite3_int64 n; - sqlite3_initialize(); - sqlite3_mutex_enter(mem0.mutex); - n = mem0.mxUsed; - if( resetFlag ){ - mem0.mxUsed = mem0.nowUsed; - } - sqlite3_mutex_leave(mem0.mutex); - return n; + int n, mx; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); + return (sqlite3_int64)mx; } /* @@ -204,7 +189,7 @@ static void sqlite3MallocAlarm(int nByte){ if( mem0.alarmCallback==0 || mem0.alarmBusy ) return; mem0.alarmBusy = 1; xCallback = mem0.alarmCallback; - nowUsed = mem0.nowUsed; + nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); pArg = mem0.alarmArg; sqlite3_mutex_leave(mem0.mutex); xCallback(pArg, nowUsed, nByte); @@ -212,6 +197,35 @@ static void sqlite3MallocAlarm(int nByte){ mem0.alarmBusy = 0; } +/* +** Do a memory allocation with statistics and alarms. Assume the +** lock is already held. +*/ +static int mallocWithAlarm(int n, void **pp){ + int nFull; + void *p; + assert( sqlite3_mutex_held(mem0.mutex) ); + nFull = sqlite3Config.m.xRoundup(n); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmCallback!=0 ){ + int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed+nFull >= mem0.alarmThreshold ){ + sqlite3MallocAlarm(nFull); + } + } + if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ){ + p = 0; + }else{ + p = sqlite3Config.m.xMalloc(nFull); + if( p==0 ){ + sqlite3MallocAlarm(nFull); + p = malloc(nFull); + } + } + if( p ) sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); + *pp = p; + return nFull; +} /* ** Allocate memory. This routine is like sqlite3_malloc() except that it @@ -219,31 +233,11 @@ static void sqlite3MallocAlarm(int nByte){ */ void *sqlite3Malloc(int n){ void *p; - int nFull; if( n<=0 ){ - return 0; + p = 0; }else if( sqlite3Config.bMemstat ){ - nFull = sqlite3Config.m.xRoundup(n); sqlite3_mutex_enter(mem0.mutex); - if( n>mem0.mxReq ) mem0.mxReq = n; - if( mem0.alarmCallback!=0 && mem0.nowUsed+nFull>=mem0.alarmThreshold ){ - sqlite3MallocAlarm(nFull); - } - if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ){ - p = 0; - }else{ - p = sqlite3Config.m.xMalloc(nFull); - if( p==0 ){ - sqlite3MallocAlarm(nFull); - p = malloc(nFull); - } - } - if( p ){ - mem0.nowUsed += nFull; - if( mem0.nowUsed>mem0.mxUsed ){ - mem0.mxUsed = mem0.nowUsed; - } - } + mallocWithAlarm(n, &p); sqlite3_mutex_leave(mem0.mutex); }else{ p = sqlite3Config.m.xMalloc(n); @@ -295,21 +289,43 @@ void *sqlite3ScratchMalloc(int n){ ** single-threaded case since checking in the multi-threaded case ** would be much more complicated.) */ assert( scratchAllocOut==0 ); - scratchAllocOut = 1; #endif - sqlite3_mutex_enter(mem0.mutex); - if( n>mem0.mxScratchReq ) mem0.mxScratchReq = n; - if( mem0.nScratchFree==0 || sqlite3Config.szScratch>=n ){ - p = sqlite3Config.m.xMalloc(n); - }else{ - int i; - i = mem0.aScratchFree[--mem0.nScratchFree]; - i *= sqlite3Config.szScratch; - p = (void*)&((char*)sqlite3Config.pScratch)[i]; + if( sqlite3Config.szScratch=(void*)mem0.aScratchFree ){ - sqlite3Config.m.xFree(p); + || p=(void*)mem0.aScratchFree ){ + if( sqlite3Config.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3Config.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3Config.m.xFree(p); + } }else{ int i; - sqlite3_mutex_enter(mem0.mutex); - assert( mem0.nScratchFree=0 && i0 ); + assert( (n & (n-1))==0 ); + assert( n>=512 && n<=32768 ); + if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ){ + return 0; + } + + if( sqlite3Config.szPage=(void*)mem0.aPageFree ){ + if( sqlite3Config.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3Config.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3Config.m.xFree(p); + } + }else{ + int i; + i = p - sqlite3Config.pPage; + i /= sqlite3Config.szPage; + assert( i>=0 && imem0.mxReq ) mem0.mxReq = nBytes; + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); nNew = sqlite3Config.m.xRoundup(nBytes); if( nOld==nNew ){ pNew = pOld; }else{ - if( mem0.nowUsed+nNew-nOld>=mem0.alarmThreshold ){ + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nNew-nOld >= + mem0.alarmThreshold ){ sqlite3MallocAlarm(nNew-nOld); } if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ){ @@ -407,10 +497,7 @@ void *sqlite3Realloc(void *pOld, int nBytes){ } } if( pNew ){ - mem0.nowUsed += nNew-nOld; - if( mem0.nowUsed>mem0.mxUsed ){ - mem0.mxUsed = mem0.nowUsed; - } + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nNew-nOld); } } sqlite3_mutex_leave(mem0.mutex); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5adaf5b666..b072c06634 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -30,7 +30,7 @@ ** the version number) and changes its name to "sqlite3.h" as ** part of the build process. ** -** @(#) $Id: sqlite.h.in,v 1.336 2008/06/18 18:57:42 danielk1977 Exp $ +** @(#) $Id: sqlite.h.in,v 1.337 2008/06/19 00:16:08 drh Exp $ */ #ifndef _SQLITE3_H_ #define _SQLITE3_H_ @@ -1057,7 +1057,7 @@ struct sqlite3_mem_methods { ** scratch memory. There are three arguments: A pointer to the memory, the ** size of each scratch buffer (sz), and the number of buffers (N). The sz ** argument must be a multiple of 16. The first -** argument should point to an allocation of at least (sz+1)*N bytes of memory. +** argument should point to an allocation of at least (sz+4)*N bytes of memory. ** SQLite will use no more than one scratch buffer at once per thread, so ** N should be set to the expected maximum number of threads. The sz ** parameter should be 6 times the size of the largest database page size. @@ -5998,6 +5998,95 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PRNG_RESET 7 #define SQLITE_TESTCTRL_BITVEC_TEST 8 +/* +** CAPI3REF: SQLite Runtime Status {F17200} +** +** This interface is used to retrieve run-time status information +** about the preformance of SQLite, and optionally to reset various +** highwater marks. The first argument is an integer code for +** the specific parameter to measure. Recognized integer codes +** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...]. +** The current value of the parameter is returned into *pCurrent. +** The highest recorded value is returned in *pHighwater. If the +** resetFlag is true, then the highest record value is reset after +** *pHighwater is written. Some parameters do not record the highest +** value. For those parameters +** nothing is written into *pHighwater and the resetFlag is ignored. +** Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent. +** +** This routine returns SQLITE_OK on success and a non-zero +** [error code] on failure. +** +** This routine is threadsafe but is not atomic. This routine can +** called while other threads are running the same or different SQLite +** interfaces. However the values returned in *pCurrent and +** *pHighwater reflect the status of SQLite at different points in time +** and it is possible that another thread might change the parameter +** in between the times when *pCurrent and *pHighwater are written. +** +** This interface is experimental and is subject to change or +** removal in future releases of SQLite. +*/ +int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); + +/* +** CAPI3REF: Status Parameters {F17250} +** +** These integer constants designate various run-time status parameters +** that can be returned by [sqlite3_status()]. +** +**
+**
SQLITE_STATUS_MEMORY_USED
+**
This parameter is the current amount of memory checked out +** using [sqlite3_malloc()], either directly or indirectly. The +** figure includes calls made to [sqlite3_malloc()] by the application +** and internal memory usage by the SQLite library. Scratch memory +** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache +** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in +** this parameter. The amount returned is the sum of the allocation +** sizes as are reported by the xSize method in [sqlite3_mem_methods].
+** +**
SQLITE_STATUS_PAGECACHE_USED
+**
This parameter returns the number of pages used out of the +** page cache buffer configured using [SQLITE_CONFIG_PAGECACHE]. The +** value returned is in pages, not in bytes.
+** +**
SQLITE_STATUS_PAGECACHE_OVERFLOW
+**
This parameter returns the number of bytes of page cache +** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** buffer and where forced to overflow to [sqlite3_malloc()].
+** +**
SQLITE_STATUS_SCRATCH_USED
+**
This parameter returns the number of allocations used out of the +** scratch allocation lookaside buffer configured using +** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not +** in bytes. Since a single thread may only have one allocation +** outstanding at time, this parameter also reports the number of threads +** using scratch memory at the same time.
+** +**
SQLITE_STATUS_PAGECACHE_OVERFLOW
+**
This parameter returns the number of bytes of scratch memory +** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** buffer and where forced to overflow to [sqlite3_malloc()].
+** +**
SQLITE_STATUS_MALLOC_SIZE
+**
This parameter records the largest memory allocation request +** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their +** internal equivalents). The value of interest is return in the +** *pHighwater parameter to [sqlite3_status()]. The value written +** into the *pCurrent parameter is undefined.
+**
+** +** New status parameters may be added from time to time. +*/ +#define SQLITE_STATUS_MEMORY_USED 0 +#define SQLITE_STATUS_PAGECACHE_USED 1 +#define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 +#define SQLITE_STATUS_SCRATCH_USED 3 +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 +#define SQLITE_STATUS_MALLOC_SIZE 5 + /* ** Undo the hack that converts floating point types to integer for diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3496200e64..7097bf60c9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.715 2008/06/18 18:57:42 danielk1977 Exp $ +** @(#) $Id: sqliteInt.h,v 1.716 2008/06/19 00:16:08 drh Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -1810,6 +1810,11 @@ sqlite3_mutex *sqlite3MutexAlloc(int); int sqlite3MutexInit(void); int sqlite3MutexEnd(void); +void sqlite3StatusReset(void); +int sqlite3StatusValue(int); +void sqlite3StatusAdd(int, int); +void sqlite3StatusSet(int, int); + int sqlite3IsNaN(double); char *sqlite3MPrintf(sqlite3*,const char*, ...); diff --git a/src/test_malloc.c b/src/test_malloc.c index 95ee365616..4643f2e55c 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -13,7 +13,7 @@ ** This file contains code used to implement test interfaces to the ** memory allocation subsystem. ** -** $Id: test_malloc.c,v 1.24 2008/06/18 18:12:04 drh Exp $ +** $Id: test_malloc.c,v 1.25 2008/06/19 00:16:08 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -669,16 +669,16 @@ static int test_config_scratch( ){ int sz, N, rc; Tcl_Obj *pResult; - static char buf[20000]; + static char buf[30000]; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "SIZE N"); return TCL_ERROR; } - if( Tcl_GetIntFromObj(interp, objv[2], &sz) ) return TCL_ERROR; - if( Tcl_GetIntFromObj(interp, objv[3], &N) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[1], &sz) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &N) ) return TCL_ERROR; if( sz<0 ){ rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, 0, 0, 0); - }else if( sz==0 ){ + }else{ int mx = sizeof(buf)/(sz+4); if( N>mx ) N = mx; rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, buf, sz, N); @@ -713,11 +713,11 @@ static int test_config_pagecache( Tcl_WrongNumArgs(interp, 1, objv, "SIZE N"); return TCL_ERROR; } - if( Tcl_GetIntFromObj(interp, objv[2], &sz) ) return TCL_ERROR; - if( Tcl_GetIntFromObj(interp, objv[3], &N) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[1], &sz) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &N) ) return TCL_ERROR; if( sz<0 ){ - rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, 0, 0, 0); - }else if( sz==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, 0, 0, 0); + }else{ int mx = sizeof(buf)/(sz+4); if( N>mx ) N = mx; rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, buf, sz, N); @@ -729,6 +729,56 @@ static int test_config_pagecache( return TCL_OK; } +/* +** Usage: sqlite3_status OPCODE RESETFLAG +** +** Return a list of three elements which are the sqlite3_status() return +** code, the current value, and the high-water mark value. +*/ +static int test_status( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc, iValue, mxValue; + int i, op, resetFlag; + const char *zOpName; + static const struct { + const char *zName; + int op; + } aOp[] = { + { "SQLITE_STATUS_MEMORY_USED", SQLITE_STATUS_MEMORY_USED }, + { "SQLITE_STATUS_PAGECACHE_USED", SQLITE_STATUS_PAGECACHE_USED }, + { "SQLITE_STATUS_PAGECACHE_OVERFLOW", SQLITE_STATUS_PAGECACHE_OVERFLOW }, + { "SQLITE_STATUS_SCRATCH_USED", SQLITE_STATUS_SCRATCH_USED }, + { "SQLITE_STATUS_SCRATCH_OVERFLOW", SQLITE_STATUS_SCRATCH_OVERFLOW }, + { "SQLITE_STATUS_MALLOC_SIZE", SQLITE_STATUS_MALLOC_SIZE }, + }; + Tcl_Obj *pResult; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "PARAMETER RESETFLAG"); + return TCL_ERROR; + } + zOpName = Tcl_GetString(objv[1]); + for(i=0; i=ArraySize(aOp) ){ + if( Tcl_GetIntFromObj(interp, objv[1], &op) ) return TCL_ERROR; + } + if( Tcl_GetBooleanFromObj(interp, objv[2], &resetFlag) ) return TCL_ERROR; + rc = sqlite3_status(op, &iValue, &mxValue, resetFlag); + pResult = Tcl_NewObj(); + Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(rc)); + Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(iValue)); + Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(mxValue)); + Tcl_SetObjResult(interp, pResult); + return TCL_OK; +} /* ** Register commands with the TCL interpreter. @@ -754,6 +804,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_memdebug_log", test_memdebug_log }, { "sqlite3_config_scratch", test_config_scratch }, { "sqlite3_config_pagecache", test_config_pagecache }, + { "sqlite3_status", test_status }, }; int i; for(i=0; i4096} +} 1 +do_test memsubsys1-5.5 { + set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] +} 0 +do_test memsubsys1-5.6 { + set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] + expr {$s_ovfl>6000} +} 1 + +# Test 6: Activate both PAGECACHE and SCRATCH with a 4k page size. +# Make it so that SCRATCH is large enough +# +db close +sqlite3_shutdown +sqlite3_config_pagecache 4096 24 +sqlite3_config_scratch 25000 1 +sqlite3_initialize +build_test_db memsubsys1-6 {PRAGMA page_size=4096} +#show_memstats +do_test memsubsys1-6.3 { + set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] +} 24 +do_test memsubsys1-6.4 { + set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] +} 4096 +do_test memsubsys1-6.5 { + set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] +} 1 +do_test memsubsys1-6.6 { + set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] +} 0 + +# Test 7: Activate both PAGECACHE and SCRATCH with a 4k page size. +# Set cache_size small so that no PAGECACHE overflow occurs. Verify +# that maximum allocation size is small. +# +db close +sqlite3_shutdown +sqlite3_config_pagecache 4096 24 +sqlite3_config_scratch 25000 1 +sqlite3_initialize +breakpoint +build_test_db memsubsys1-7 { + PRAGMA page_size=4096; + PRAGMA cache_size=10; + PRAGMA temp_cache_size=10; +} +#show_memstats +do_test memsubsys1-7.3 { + set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] + expr {$pg_used<24} +} 1 +do_test memsubsys1-7.4 { + set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] +} 0 +do_test memsubsys1-7.5 { + set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] + expr {$maxreq<2500} +} 1 +do_test memsubsys1-7.6 { + set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] +} 1 +do_test memsubsys1-7.7 { + set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] +} 0 + + +finish_test diff --git a/test/tester.tcl b/test/tester.tcl index a3dd224a81..e014732ccc 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -11,7 +11,7 @@ # This file implements some common TCL routines used for regression # testing the SQLite library # -# $Id: tester.tcl,v 1.126 2008/06/07 05:19:38 danielk1977 Exp $ +# $Id: tester.tcl,v 1.127 2008/06/19 00:16:08 drh Exp $ # # What for user input before continuing. This gives an opportunity @@ -333,6 +333,7 @@ proc finalize_testing {} { sqlite3_memdebug_dump ./memusage.txt } } + show_memstats puts "Maximum memory usage: [sqlite3_memory_highwater 1] bytes" puts "Current memory usage: [sqlite3_memory_highwater] bytes" if {[info commands sqlite3_memdebug_malloc_count] ne ""} { @@ -359,6 +360,28 @@ proc finalize_testing {} { exit [expr {$nErr>0}] } +# Display memory statistics for analysis and debugging purposes. +# +proc show_memstats {} { + set x [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] + set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] + puts "Memory used: $val" + set x [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] + set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] + puts "Page-cache used: $val" + set x [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] + set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] + puts "Page-cache overflow: $val" + set x [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] + set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] + puts "Scratch memory used: $val" + set x [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] + set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] + puts "Scratch overflow: $val" + set x [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] + puts "Maximum alloc size: [lindex $x 2]" +} + # A procedure to execute SQL # proc execsql {sql {db db}} { diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index ea6034c82c..4c1d0ba387 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -204,6 +204,7 @@ foreach file { sqliteInt.h global.c + status.c date.c os.c