From 634f298c89027c3d6ea91f2a2893909ae2594859 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Mon, 28 Mar 2005 08:44:07 +0000 Subject: [PATCH] Modifications to reduce memory consumption. (CVS 2422) FossilOrigin-Name: 0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2 --- manifest | 20 +++---- manifest.uuid | 2 +- src/btree.c | 67 +++++++++++++--------- src/pager.c | 13 ++++- src/vdbeaux.c | 41 ++++++++++++-- tool/memleak3.tcl | 141 +++++++++++++++++++++++++++++++++++++++------- 6 files changed, 218 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index cf91940a0d..2cdf45b3e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\smemory\sleaks\sthat\soccur\safter\sa\smalloc\sfailure.\s(CVS\s2421) -D 2005-03-28T03:39:56 +C Modifications\sto\sreduce\smemory\sconsumption.\s(CVS\s2422) +D 2005-03-28T08:44:07 F Makefile.in 5c00d0037104de2a50ac7647a5f12769795957a3 F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -30,7 +30,7 @@ F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a F src/alter.c 9570af388bc99471ea6e1258817fbf06e3120030 F src/attach.c 3615dbe960cbee4aa5ea300b8a213dad36527b0f F src/auth.c 18c5a0befe20f3a58a41e3ddd78f372faeeefe1f -F src/btree.c c33c0e6833eb8ac0f0941c1f8e722f7c70dbef57 +F src/btree.c 657fd61dfb4dd37a63416652b1a28700e84b36f7 F src/btree.h 41a71ce027db9ddee72cb43df2316bbe3a1d92af F src/build.c 2589c2ffa263406526d0cc5728405c6c2f9638f6 F src/date.c 2134ef4388256e8247405178df8a61bd60dc180a @@ -52,7 +52,7 @@ F src/os_unix.c fba0167576f09e242afd4c4978e1d2944b1da8b5 F src/os_unix.h 40b2fd1d02cfa45d6c3dea25316fd019cf9fcb0c F src/os_win.c 2bbbe6fbb010763c3fa79d5e951afca9b138c6b5 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c 4c1322dc8458652eb61d23405edd07a7a201160b +F src/pager.c 221076cde9af787fc45eec1bfebf5f20e4faacf0 F src/pager.h 9a417a1e04737c227ebbba3bdf8597d6dd51513a F src/parse.y 10c0ace9efce31d5a06e03488a4284b9d97abc56 F src/pragma.c 4b20dbc0f4b97f412dc511853d3d0c2e0d4adedc @@ -79,7 +79,7 @@ F src/vdbe.c c7973dc0ab52538646018620e3d3c68aa9e6d6c4 F src/vdbe.h 7f586cb6d6b57764e5aac1f87107d6a95ddce24c F src/vdbeInt.h e80721cd8ff611789e20743eec43363a9fb5a48e F src/vdbeapi.c 467caa6e6fb9247528b1c7ab9132ae1b4748e8ac -F src/vdbeaux.c 91013922626fed75ad091459b7f05f9e86581690 +F src/vdbeaux.c 0932f570d276992c7b3ee989589b6ff9056f97e7 F src/vdbemem.c 4e853ce3151eaf7906150da85a1b3ce1fe5e8da8 F src/where.c c4b227458e8993decb515ed9a2fe2d4f5f8e3125 F tclinstaller.tcl 046e3624671962dc50f0481d7c25b38ef803eb42 @@ -221,7 +221,7 @@ F tool/lemon.c c88936c67f6411608db8fa4254d254f509fa40f6 F tool/lempar.c e8b0eb00a6b905ce2ebd55965ed243574482cd5f F tool/memleak.awk 4e7690a51bf3ed757e611273d43fe3f65b510133 F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8 -F tool/memleak3.tcl b8eb053190e95a55dc188896afb972e8108822d6 +F tool/memleak3.tcl 2b1ab290badf3b26f9ba433baf7fad8def14aea8 F tool/mkkeywordhash.c 02ac5c523fd6d55364cd70aded5c36ba6651a6bf F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -278,7 +278,7 @@ F www/tclsqlite.tcl e73f8f8e5f20e8277619433f7970060ab01088fc F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0 F www/whentouse.tcl 528299b8316726dbcc5548e9aa0648c8b1bd055b -P ccb9f4022b3ccb1cc2ab001628fd38becfbf8efe -R ceb3b5e5af33f6f582749ad4190a9e4b -U drh -Z feff361f817d89fa1dffb88a9f3296a3 +P bcb5d72ef146b1019c72220701d385c7b0b5d0bd +R bb1ad29712ee81032569962fcc3720e5 +U danielk1977 +Z f1d40042415d09b383776e4b14cf65b8 diff --git a/manifest.uuid b/manifest.uuid index c3a409536c..5cbd01e8c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcb5d72ef146b1019c72220701d385c7b0b5d0bd \ No newline at end of file +0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b2c34e533f..49fe8d9dac 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.253 2005/03/21 04:04:02 danielk1977 Exp $ +** $Id: btree.c,v 1.254 2005/03/28 08:44:07 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -3757,7 +3757,8 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){ static int balance_nonroot(MemPage *pPage){ MemPage *pParent; /* The parent of pPage */ Btree *pBt; /* The whole database */ - int nCell = 0; /* Number of cells in aCell[] */ + int nCell = 0; /* Number of cells in apCell[] */ + int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ int nOld; /* Number of pages in apOld[] */ int nNew; /* Number of pages in apNew[] */ int nDiv; /* Number of cells in apDiv[] */ @@ -3771,7 +3772,6 @@ static int balance_nonroot(MemPage *pPage){ int pageFlags; /* Value of pPage->aData[0] */ int subtotal; /* Subtotal of bytes in cells on one page */ int iSpace = 0; /* First unused byte of aSpace[] */ - int mxCellPerPage; /* Maximum number of cells in one page */ MemPage *apOld[NB]; /* pPage and up to two siblings */ Pgno pgnoOld[NB]; /* Page numbers for each page in apOld[] */ MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ @@ -3825,31 +3825,6 @@ static int balance_nonroot(MemPage *pPage){ } #endif - /* - ** Allocate space for memory structures - */ - mxCellPerPage = MX_CELL(pBt); - apCell = sqliteMallocRaw( - (mxCellPerPage+2)*NB*(sizeof(u8*)+sizeof(int)) - + sizeof(MemPage)*NB - + pBt->psAligned*(5+NB) - + (ISAUTOVACUUM ? (mxCellPerPage+2)*NN*2 : 0) - ); - if( apCell==0 ){ - return SQLITE_NOMEM; - } - szCell = (int*)&apCell[(mxCellPerPage+2)*NB]; - aCopy[0] = (u8*)&szCell[(mxCellPerPage+2)*NB]; - for(i=1; ipsAligned+sizeof(MemPage)]; - } - aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)]; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - aFrom = &aSpace[5*pBt->psAligned]; - } -#endif - /* ** Find the cell in the parent page whose left child points back ** to pPage. The "idx" variable is the index of that cell. If pPage @@ -3910,8 +3885,35 @@ static int balance_nonroot(MemPage *pPage){ apCopy[i] = 0; assert( i==nOld ); nOld++; + nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; } + /* + ** Allocate space for memory structures + */ + apCell = sqliteMallocRaw( + nMaxCells*sizeof(u8*) /* apCell */ + + nMaxCells*sizeof(int) /* szCell */ + + sizeof(MemPage)*NB /* aCopy */ + + pBt->psAligned*(5+NB) /* aSpace */ + + (ISAUTOVACUUM ? nMaxCells : 0) /* aFrom */ + ); + if( apCell==0 ){ + rc = SQLITE_NOMEM; + goto balance_cleanup; + } + szCell = (int*)&apCell[nMaxCells]; + aCopy[0] = (u8*)&szCell[nMaxCells]; + for(i=1; ipsAligned+sizeof(MemPage)]; + } + aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)]; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + aFrom = &aSpace[5*pBt->psAligned]; + } +#endif + /* ** Make copies of the content of pPage and its siblings into aOld[]. ** The rest of this function will use data from the copies rather @@ -3948,6 +3950,7 @@ static int balance_nonroot(MemPage *pPage){ MemPage *pOld = apCopy[i]; int limit = pOld->nCell+pOld->nOverflow; for(j=0; jusableSize - 12 + leafCorrection; for(subtotal=k=i=0; i usableSpace ){ szNew[k] = subtotal - szCell[i]; @@ -4051,6 +4056,8 @@ static int balance_nonroot(MemPage *pPage){ r = cntNew[i-1] - 1; d = r + 1 - leafData; + assert( dpgno==pgnoNew[i] ); assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]); @@ -4160,6 +4168,7 @@ static int balance_nonroot(MemPage *pPage){ */ if( pBt->autoVacuum ){ for(k=j; kpgno!=pNew->pgno ){ rc = ptrmapPutOvfl(pNew, k-j); if( rc!=SQLITE_OK ){ @@ -4179,6 +4188,8 @@ static int balance_nonroot(MemPage *pPage){ u8 *pCell; u8 *pTemp; int sz; + + assert( jleaf ){ diff --git a/src/pager.c b/src/pager.c index 9cc1bb0256..f84987e33c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.199 2005/03/28 03:39:56 drh Exp $ +** @(#) $Id: pager.c,v 1.200 2005/03/28 08:44:07 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -213,9 +213,16 @@ struct PgHistory { /* ** How big to make the hash table used for locating in-memory pages -** by page number. +** by page number. This macro looks a little silly, but is evaluated +** at compile-time, not run-time (at least for gcc this is true). */ -#define N_PG_HASH 2048 +#define N_PG_HASH (\ + (MAX_PAGES>1024)?2048: \ + (MAX_PAGES>512)?1024: \ + (MAX_PAGES>256)?512: \ + (MAX_PAGES>128)?256: \ + (MAX_PAGES>64)?128:64 \ +) /* ** Hash a page number diff --git a/src/vdbeaux.c b/src/vdbeaux.c index fa67a41abf..d2c934063a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -57,10 +57,16 @@ void sqlite3VdbeTrace(Vdbe *p, FILE *trace){ /* ** Resize the Vdbe.aOp array so that it contains at least N +** elements. If the Vdbe is in VDBE_MAGIC_RUN state, then +** the Vdbe.aOp array will be sized to contain exactly N ** elements. */ static void resizeOpArray(Vdbe *p, int N){ - if( p->nOpAllocmagic==VDBE_MAGIC_RUN ){ + assert( N==p->nOp ); + p->nOpAlloc = N; + p->aOp = sqliteRealloc(p->aOp, N*sizeof(Op)); + }else if( p->nOpAllocnOpAlloc; p->nOpAlloc = N+100; p->aOp = sqliteRealloc(p->aOp, p->nOpAlloc*sizeof(Op)); @@ -166,19 +172,35 @@ void sqlite3VdbeResolveLabel(Vdbe *p, int x){ ** value to its correct non-zero value. ** ** This routine is called once after all opcodes have been inserted. +** +** Variable *pMaxFuncArgs is set to the maximum value of any P1 argument +** to an OP_Function or P2 to an OP_AggFunc opcode. This is used by +** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. */ -static void resolveP2Values(Vdbe *p){ +static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ int i; + int nMax = 0; Op *pOp; int *aLabel = p->aLabel; if( aLabel==0 ) return; for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ + u8 opcode = pOp->opcode; + + /* Todo: Maybe OP_AggFunc should change to use P1 in the same + * way as OP_Function. */ + if( opcode==OP_Function ){ + if( pOp->p1>nMax ) nMax = pOp->p1; + }else if( opcode==OP_AggFunc ){ + if( pOp->p2>nMax ) nMax = pOp->p2; + } + if( pOp->p2>=0 ) continue; assert( -1-pOp->p2nLabel ); pOp->p2 = aLabel[-1-pOp->p2]; } sqliteFree(p->aLabel); p->aLabel = 0; + *pMaxFuncArgs = nMax; } /* @@ -603,6 +625,13 @@ void sqlite3VdbeMakeReady( */ assert( p->nOp>0 ); + /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. This + * is because the call to resizeOpArray() below may shrink the + * p->aOp[] array to save memory if called when in VDBE_MAGIC_RUN + * state. + */ + p->magic = VDBE_MAGIC_RUN; + /* No instruction ever pushes more than a single element onto the ** stack. And the stack never grows on successive executions of the ** same loop. So the total number of instructions is an upper bound @@ -611,12 +640,14 @@ void sqlite3VdbeMakeReady( ** Allocation all the stack space we will ever need. */ if( p->aStack==0 ){ - resolveP2Values(p); + int nArg; /* Maximum number of args passed to a user function. */ + resolveP2Values(p, &nArg); + resizeOpArray(p, p->nOp); assert( nVar>=0 ); n = isExplain ? 10 : p->nOp; p->aStack = sqliteMalloc( n*sizeof(p->aStack[0]) /* aStack */ - + n*sizeof(Mem*) /* apArg */ + + nArg*sizeof(Mem*) /* apArg */ + nVar*sizeof(Mem) /* aVar */ + nVar*sizeof(char*) /* azVar */ + nMem*sizeof(Mem) /* aMem */ @@ -630,7 +661,7 @@ void sqlite3VdbeMakeReady( p->nVar = nVar; p->okVar = 0; p->apArg = (Mem**)&p->aVar[nVar]; - p->azVar = (char**)&p->apArg[n]; + p->azVar = (char**)&p->apArg[nArg]; p->apCsr = (Cursor**)&p->azVar[nVar]; if( nAgg>0 ){ p->nAgg = nAgg; diff --git a/tool/memleak3.tcl b/tool/memleak3.tcl index f7fa5b43b2..d3fde1ee06 100644 --- a/tool/memleak3.tcl +++ b/tool/memleak3.tcl @@ -25,23 +25,60 @@ If all goes well a summary of unfreed allocations is printed out. If the GNU C library is in use and SQLITE_DEBUG is 3 or greater a stack trace is printed out for each unmatched allocation. +If the \"-r \" option is passed, then the program stops and prints out +the state of the heap immediately after the th call to malloc() or +realloc(). + Example: $ ./testfixture ../sqlite/test/select1.test 2> memtrace.out -$ tclsh $argv0 ./testfixture memtrace.out +$ tclsh $argv0 ?-r ? ./testfixture memtrace.out " - -# If stack traces are enabled, the 'addr2line' program is called to -# translate a binary stack address into a human-readable form. -set addr2line addr2line - -if { [llength $argv]!=2 } { - puts "Usage: $argv0 " +if { [llength $argv]!=2 && [llength $argv]!=4 } { + set prg [file tail $argv0] + puts "Usage: $prg ?-r ? " puts "" puts [string trim $doco] exit -1 } +# If stack traces are enabled, the 'addr2line' program is called to +# translate a binary stack address into a human-readable form. +set addr2line addr2line + +# When the SQLITE_MEMDEBUG is set as described above, SQLite prints +# out a line for each malloc(), realloc() or free() call that the +# library makes. If SQLITE_MEMDEBUG is 3, then a stack trace is printed +# out before each malloc() and realloc() line. +# +# This program parses each line the SQLite library outputs and updates +# the following global Tcl variables to reflect the "current" state of +# the heap used by SQLite. +# +set nBytes 0 ;# Total number of bytes currently allocated. +set nMalloc 0 ;# Total number of malloc()/realloc() calls. +set nPeak 0 ;# Peak of nBytes. +set iPeak 0 ;# nMalloc when nPeak was set. +# +# More detailed state information is stored in the $memmap array. +# Each key in the memmap array is the address of a chunk of memory +# currently allocated from the heap. The value is a list of the +# following form +# +# { } +# +array unset memmap + +# The executable program being analyzed. +if {[llength $argv]==2} { + set exe [lindex $argv 0] + set memfile [lindex $argv 1] + set report_at -1 +} else { + set exe [lindex $argv 2] + set memfile [lindex $argv 3] + set report_at [lindex $argv 1] +} proc process_input {input_file array_name} { upvar $array_name mem @@ -68,6 +105,17 @@ proc process_input {input_file array_name} { set mem($addr) [list $bytes "malloc $mallocid" $stack] set stack "" + # Increase the current heap usage + incr ::nBytes $bytes + + # Increase the number of malloc() calls + incr ::nMalloc + + if {$::nBytes > $::nPeak} { + set ::nPeak $::nBytes + set ::iPeak $::nMalloc + } + } elseif { [regexp $FREE $line dummy bytes addr] } { # If this is a 'free' line, remove the entry from the mem array. If the # entry does not exist, or is the wrong number of bytes, announce a @@ -78,31 +126,86 @@ proc process_input {input_file array_name} { } unset mem($addr) + # Decrease the current heap usage + incr ::nBytes [expr -1 * $bytes] + } elseif { [regexp $REALLOC $line dummy mallocid ob b oa a] } { - # If it is a realloc line, remove the old mem entry and add a new one. + # "free" the old allocation in the internal model: + incr ::nBytes [expr -1 * $ob] unset mem($oa); + + # "malloc" the new allocation set mem($a) [list $b "realloc $mallocid" $stack] + incr ::nBytes $b set stack "" + + # Increase the number of malloc() calls + incr ::nMalloc + + if {$::nBytes > $::nPeak} { + set ::nPeak $::nBytes + set ::iPeak $::nMalloc + } + } else { # puts "REJECT: $line" } + + if {$::nMalloc==$::report_at} report } close $input } -process_input [lindex $argv 1] mem -set exe [lindex $argv 0] - -foreach key [array names mem] { - set bytes [lindex $mem($key) 0] - set mallocid [lindex $mem($key) 1] - set stack [lindex $mem($key) 2] - puts "Leaked $bytes bytes at 0x$key: $mallocid" - foreach frame [lrange $stack 1 10] { - foreach {f l} [split [exec $addr2line -f --exe=$exe $frame] \n] {} +proc printstack {stack} { + set fcount 10 + if {[llength $stack]<10} { + set fcount [llength $stack] + } + foreach frame [lrange $stack 1 $fcount] { + foreach {f l} [split [exec $::addr2line -f --exe=$::exe $frame] \n] {} puts [format "%-30s %s" $f $l] } if {[llength $stack]>0 } {puts ""} } +proc report {} { + + foreach key [array names ::memmap] { + set stack [lindex $::memmap($key) 2] + set bytes [lindex $::memmap($key) 0] + lappend summarymap($stack) $bytes + } + + foreach stack [array names summarymap] { + set allocs $summarymap($stack) + set sum 0 + foreach a $allocs { + incr sum $a + } + lappend sorted [list $sum $stack] + } + + set sorted [lsort -integer -index 0 $sorted] + foreach s $sorted { + set sum [lindex $s 0] + set stack [lindex $s 1] + set allocs $summarymap($stack) + puts "$sum bytes in [llength $allocs] chunks ($allocs)" + printstack $stack + } + + # Print out summary statistics + puts "Total allocations : $::nMalloc" + puts "Total outstanding allocations: [array size ::memmap]" + puts "Current heap usage : $::nBytes bytes" + puts "Peak heap usage : $::nPeak bytes (malloc #$::iPeak)" + + exit +} + +process_input $memfile memmap +report + + +