mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modifications to reduce memory consumption. (CVS 2422)

Browse Source 
FossilOrigin-Name: 0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2
This commit is contained in:
danielk1977 2005-03-28 08:44:07 +00:00
parent dd5b2fa5f2
commit 634f298c89
6 changed files with 218 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
manifest
View File

@ -1,5 +1,5 @@
C Fix\ssome\smemory\sleaks\sthat\soccur\safter\sa\smalloc\sfailure.\s(CVS\s2421)
D 2005-03-28T03:39:56
C Modifications\sto\sreduce\smemory\sconsumption.\s(CVS\s2422)
D 2005-03-28T08:44:07
F Makefile.in 5c00d0037104de2a50ac7647a5f12769795957a3
F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@ -30,7 +30,7 @@ F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a
F src/alter.c 9570af388bc99471ea6e1258817fbf06e3120030
F src/attach.c 3615dbe960cbee4aa5ea300b8a213dad36527b0f
F src/auth.c 18c5a0befe20f3a58a41e3ddd78f372faeeefe1f
F src/btree.c c33c0e6833eb8ac0f0941c1f8e722f7c70dbef57
F src/btree.c 657fd61dfb4dd37a63416652b1a28700e84b36f7
F src/btree.h 41a71ce027db9ddee72cb43df2316bbe3a1d92af
F src/build.c 2589c2ffa263406526d0cc5728405c6c2f9638f6
F src/date.c 2134ef4388256e8247405178df8a61bd60dc180a
@ -52,7 +52,7 @@ F src/os_unix.c fba0167576f09e242afd4c4978e1d2944b1da8b5
F src/os_unix.h 40b2fd1d02cfa45d6c3dea25316fd019cf9fcb0c
F src/os_win.c 2bbbe6fbb010763c3fa79d5e951afca9b138c6b5
F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
F src/pager.c 4c1322dc8458652eb61d23405edd07a7a201160b
F src/pager.c 221076cde9af787fc45eec1bfebf5f20e4faacf0
F src/pager.h 9a417a1e04737c227ebbba3bdf8597d6dd51513a
F src/parse.y 10c0ace9efce31d5a06e03488a4284b9d97abc56
F src/pragma.c 4b20dbc0f4b97f412dc511853d3d0c2e0d4adedc
@ -79,7 +79,7 @@ F src/vdbe.c c7973dc0ab52538646018620e3d3c68aa9e6d6c4
F src/vdbe.h 7f586cb6d6b57764e5aac1f87107d6a95ddce24c
F src/vdbeInt.h e80721cd8ff611789e20743eec43363a9fb5a48e
F src/vdbeapi.c 467caa6e6fb9247528b1c7ab9132ae1b4748e8ac
F src/vdbeaux.c 91013922626fed75ad091459b7f05f9e86581690
F src/vdbeaux.c 0932f570d276992c7b3ee989589b6ff9056f97e7
F src/vdbemem.c 4e853ce3151eaf7906150da85a1b3ce1fe5e8da8
F src/where.c c4b227458e8993decb515ed9a2fe2d4f5f8e3125
F tclinstaller.tcl 046e3624671962dc50f0481d7c25b38ef803eb42
@ -221,7 +221,7 @@ F tool/lemon.c c88936c67f6411608db8fa4254d254f509fa40f6
F tool/lempar.c e8b0eb00a6b905ce2ebd55965ed243574482cd5f
F tool/memleak.awk 4e7690a51bf3ed757e611273d43fe3f65b510133
F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8
F tool/memleak3.tcl b8eb053190e95a55dc188896afb972e8108822d6
F tool/memleak3.tcl 2b1ab290badf3b26f9ba433baf7fad8def14aea8
F tool/mkkeywordhash.c 02ac5c523fd6d55364cd70aded5c36ba6651a6bf
F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
@ -278,7 +278,7 @@ F www/tclsqlite.tcl e73f8f8e5f20e8277619433f7970060ab01088fc
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
F www/whentouse.tcl 528299b8316726dbcc5548e9aa0648c8b1bd055b
P ccb9f4022b3ccb1cc2ab001628fd38becfbf8efe
R ceb3b5e5af33f6f582749ad4190a9e4b
U drh
Z feff361f817d89fa1dffb88a9f3296a3
P bcb5d72ef146b1019c72220701d385c7b0b5d0bd
R bb1ad29712ee81032569962fcc3720e5
U danielk1977
Z f1d40042415d09b383776e4b14cf65b8

2
manifest.uuid
View File

@ -1 +1 @@
bcb5d72ef146b1019c72220701d385c7b0b5d0bd
0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2

67
src/btree.c
View File

@ -9,7 +9,7 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.253 2005/03/21 04:04:02 danielk1977 Exp $
** $Id: btree.c,v 1.254 2005/03/28 08:44:07 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
@ -3757,7 +3757,8 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){
static int balance_nonroot(MemPage *pPage){
MemPage *pParent; /* The parent of pPage */
Btree *pBt; /* The whole database */
int nCell = 0; /* Number of cells in aCell[] */
int nCell = 0; /* Number of cells in apCell[] */
int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */
int nOld; /* Number of pages in apOld[] */
int nNew; /* Number of pages in apNew[] */
int nDiv; /* Number of cells in apDiv[] */
@ -3771,7 +3772,6 @@ static int balance_nonroot(MemPage *pPage){
int pageFlags; /* Value of pPage->aData[0] */
int subtotal; /* Subtotal of bytes in cells on one page */
int iSpace = 0; /* First unused byte of aSpace[] */
int mxCellPerPage; /* Maximum number of cells in one page */
MemPage *apOld[NB]; /* pPage and up to two siblings */
Pgno pgnoOld[NB]; /* Page numbers for each page in apOld[] */
MemPage *apCopy[NB]; /* Private copies of apOld[] pages */
@ -3825,31 +3825,6 @@ static int balance_nonroot(MemPage *pPage){
}
#endif
/*
** Allocate space for memory structures
*/
mxCellPerPage = MX_CELL(pBt);
apCell = sqliteMallocRaw(
(mxCellPerPage+2)*NB*(sizeof(u8*)+sizeof(int))
+ sizeof(MemPage)*NB
+ pBt->psAligned*(5+NB)
+ (ISAUTOVACUUM ? (mxCellPerPage+2)*NN*2 : 0)
);
if( apCell==0 ){
return SQLITE_NOMEM;
}
szCell = (int*)&apCell[(mxCellPerPage+2)*NB];
aCopy[0] = (u8*)&szCell[(mxCellPerPage+2)*NB];
for(i=1; i<NB; i++){
aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)];
}
aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)];
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum ){
aFrom = &aSpace[5*pBt->psAligned];
}
#endif
/*
** Find the cell in the parent page whose left child points back
** to pPage. The "idx" variable is the index of that cell. If pPage
@ -3910,8 +3885,35 @@ static int balance_nonroot(MemPage *pPage){
apCopy[i] = 0;
assert( i==nOld );
nOld++;
nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow;
}
/*
** Allocate space for memory structures
*/
apCell = sqliteMallocRaw(
nMaxCells*sizeof(u8*) /* apCell */
+ nMaxCells*sizeof(int) /* szCell */
+ sizeof(MemPage)*NB /* aCopy */
+ pBt->psAligned*(5+NB) /* aSpace */
+ (ISAUTOVACUUM ? nMaxCells : 0) /* aFrom */
);
if( apCell==0 ){
rc = SQLITE_NOMEM;
goto balance_cleanup;
}
szCell = (int*)&apCell[nMaxCells];
aCopy[0] = (u8*)&szCell[nMaxCells];
for(i=1; i<NB; i++){
aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)];
}
aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)];
#ifndef SQLITE_OMIT_AUTOVACUUM
if( pBt->autoVacuum ){
aFrom = &aSpace[5*pBt->psAligned];
}
#endif
/*
** Make copies of the content of pPage and its siblings into aOld[].
** The rest of this function will use data from the copies rather
@ -3948,6 +3950,7 @@ static int balance_nonroot(MemPage *pPage){
MemPage *pOld = apCopy[i];
int limit = pOld->nCell+pOld->nOverflow;
for(j=0; j<limit; j++){
assert( nCell<nMaxCells );
apCell[nCell] = findOverflowCell(pOld, j);
szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
#ifndef SQLITE_OMIT_AUTOVACUUM
@ -3975,6 +3978,7 @@ static int balance_nonroot(MemPage *pPage){
dropCell(pParent, nxDiv, sz);
}else{
u8 *pTemp;
assert( nCell<nMaxCells );
szCell[nCell] = sz;
pTemp = &aSpace[iSpace];
iSpace += sz;
@ -4020,6 +4024,7 @@ static int balance_nonroot(MemPage *pPage){
*/
usableSpace = pBt->usableSize - 12 + leafCorrection;
for(subtotal=k=i=0; i<nCell; i++){
assert( i<nMaxCells );
subtotal += szCell[i] + 2;
if( subtotal > usableSpace ){
szNew[k] = subtotal - szCell[i];
@ -4051,6 +4056,8 @@ static int balance_nonroot(MemPage *pPage){
r = cntNew[i-1] - 1;
d = r + 1 - leafData;
assert( d<nMaxCells );
assert( r<nMaxCells );
while( szRight==0 || szRight+szCell[d]+2<=szLeft-(szCell[r]+2) ){
szRight += szCell[d] + 2;
szLeft -= szCell[r] + 2;
@ -4146,6 +4153,7 @@ static int balance_nonroot(MemPage *pPage){
j = 0;
for(i=0; i<nNew; i++){
/* Assemble the new sibling page. */
assert( j<nMaxCells );
MemPage *pNew = apNew[i];
assert( pNew->pgno==pgnoNew[i] );
assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
@ -4160,6 +4168,7 @@ static int balance_nonroot(MemPage *pPage){
*/
if( pBt->autoVacuum ){
for(k=j; k<cntNew[i]; k++){
assert( k<nMaxCells );
if( aFrom[k]==0xFF || apCopy[aFrom[k]]->pgno!=pNew->pgno ){
rc = ptrmapPutOvfl(pNew, k-j);
if( rc!=SQLITE_OK ){
@ -4179,6 +4188,8 @@ static int balance_nonroot(MemPage *pPage){
u8 *pCell;
u8 *pTemp;
int sz;
assert( j<nMaxCells );
pCell = apCell[j];
sz = szCell[j] + leafCorrection;
if( !pNew->leaf ){

13
src/pager.c
View File

@ -18,7 +18,7 @@
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.199 2005/03/28 03:39:56 drh Exp $
** @(#) $Id: pager.c,v 1.200 2005/03/28 08:44:07 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
@ -213,9 +213,16 @@ struct PgHistory {
/*
** How big to make the hash table used for locating in-memory pages
** by page number.
** by page number. This macro looks a little silly, but is evaluated
** at compile-time, not run-time (at least for gcc this is true).
*/
#define N_PG_HASH 2048
#define N_PG_HASH (\
(MAX_PAGES>1024)?2048: \
(MAX_PAGES>512)?1024: \
(MAX_PAGES>256)?512: \
(MAX_PAGES>128)?256: \
(MAX_PAGES>64)?128:64 \
)
/*
** Hash a page number

41
src/vdbeaux.c
View File

@ -57,10 +57,16 @@ void sqlite3VdbeTrace(Vdbe *p, FILE *trace){
/*
** Resize the Vdbe.aOp array so that it contains at least N
** elements. If the Vdbe is in VDBE_MAGIC_RUN state, then
** the Vdbe.aOp array will be sized to contain exactly N
** elements.
*/
static void resizeOpArray(Vdbe *p, int N){
if( p->nOpAlloc<N ){
if( p->magic==VDBE_MAGIC_RUN ){
assert( N==p->nOp );
p->nOpAlloc = N;
p->aOp = sqliteRealloc(p->aOp, N*sizeof(Op));
}else if( p->nOpAlloc<N ){
int oldSize = p->nOpAlloc;
p->nOpAlloc = N+100;
p->aOp = sqliteRealloc(p->aOp, p->nOpAlloc*sizeof(Op));
@ -166,19 +172,35 @@ void sqlite3VdbeResolveLabel(Vdbe *p, int x){
** value to its correct non-zero value.
**
** This routine is called once after all opcodes have been inserted.
**
** Variable *pMaxFuncArgs is set to the maximum value of any P1 argument
** to an OP_Function or P2 to an OP_AggFunc opcode. This is used by
** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
*/
static void resolveP2Values(Vdbe *p){
static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
int i;
int nMax = 0;
Op *pOp;
int *aLabel = p->aLabel;
if( aLabel==0 ) return;
for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
u8 opcode = pOp->opcode;
/* Todo: Maybe OP_AggFunc should change to use P1 in the same
* way as OP_Function. */
if( opcode==OP_Function ){
if( pOp->p1>nMax ) nMax = pOp->p1;
}else if( opcode==OP_AggFunc ){
if( pOp->p2>nMax ) nMax = pOp->p2;
}
if( pOp->p2>=0 ) continue;
assert( -1-pOp->p2<p->nLabel );
pOp->p2 = aLabel[-1-pOp->p2];
}
sqliteFree(p->aLabel);
p->aLabel = 0;
*pMaxFuncArgs = nMax;
}
/*
@ -603,6 +625,13 @@ void sqlite3VdbeMakeReady(
*/
assert( p->nOp>0 );
/* Set the magic to VDBE_MAGIC_RUN sooner rather than later. This
* is because the call to resizeOpArray() below may shrink the
* p->aOp[] array to save memory if called when in VDBE_MAGIC_RUN
* state.
*/
p->magic = VDBE_MAGIC_RUN;
/* No instruction ever pushes more than a single element onto the
** stack. And the stack never grows on successive executions of the
** same loop. So the total number of instructions is an upper bound
@ -611,12 +640,14 @@ void sqlite3VdbeMakeReady(
** Allocation all the stack space we will ever need.
*/
if( p->aStack==0 ){
resolveP2Values(p);
int nArg; /* Maximum number of args passed to a user function. */
resolveP2Values(p, &nArg);
resizeOpArray(p, p->nOp);
assert( nVar>=0 );
n = isExplain ? 10 : p->nOp;
p->aStack = sqliteMalloc(
n*sizeof(p->aStack[0]) /* aStack */
+ n*sizeof(Mem*) /* apArg */
+ nArg*sizeof(Mem*) /* apArg */
+ nVar*sizeof(Mem) /* aVar */
+ nVar*sizeof(char*) /* azVar */
+ nMem*sizeof(Mem) /* aMem */
@ -630,7 +661,7 @@ void sqlite3VdbeMakeReady(
p->nVar = nVar;
p->okVar = 0;
p->apArg = (Mem**)&p->aVar[nVar];
p->azVar = (char**)&p->apArg[n];
p->azVar = (char**)&p->apArg[nArg];
p->apCsr = (Cursor**)&p->azVar[nVar];
if( nAgg>0 ){
p->nAgg = nAgg;

141
tool/memleak3.tcl
View File

@ -25,23 +25,60 @@ If all goes well a summary of unfreed allocations is printed out. If the
GNU C library is in use and SQLITE_DEBUG is 3 or greater a stack trace is
printed out for each unmatched allocation.
If the \"-r <n>\" option is passed, then the program stops and prints out
the state of the heap immediately after the <n>th call to malloc() or
realloc().
Example:
$ ./testfixture ../sqlite/test/select1.test 2> memtrace.out
$ tclsh $argv0 ./testfixture memtrace.out
$ tclsh $argv0 ?-r <malloc-number>? ./testfixture memtrace.out
"
# If stack traces are enabled, the 'addr2line' program is called to
# translate a binary stack address into a human-readable form.
set addr2line addr2line
if { [llength $argv]!=2 } {
puts "Usage: $argv0 <binary file> <mem trace file>"
if { [llength $argv]!=2 && [llength $argv]!=4 } {
set prg [file tail $argv0]
puts "Usage: $prg ?-r <malloc-number>? <binary file> <mem trace file>"
puts ""
puts [string trim $doco]
exit -1
}
# If stack traces are enabled, the 'addr2line' program is called to
# translate a binary stack address into a human-readable form.
set addr2line addr2line
# When the SQLITE_MEMDEBUG is set as described above, SQLite prints
# out a line for each malloc(), realloc() or free() call that the
# library makes. If SQLITE_MEMDEBUG is 3, then a stack trace is printed
# out before each malloc() and realloc() line.
#
# This program parses each line the SQLite library outputs and updates
# the following global Tcl variables to reflect the "current" state of
# the heap used by SQLite.
#
set nBytes 0 ;# Total number of bytes currently allocated.
set nMalloc 0 ;# Total number of malloc()/realloc() calls.
set nPeak 0 ;# Peak of nBytes.
set iPeak 0 ;# nMalloc when nPeak was set.
#
# More detailed state information is stored in the $memmap array.
# Each key in the memmap array is the address of a chunk of memory
# currently allocated from the heap. The value is a list of the
# following form
#
# {<number-of-bytes> <malloc id> <stack trace>}
#
array unset memmap
# The executable program being analyzed.
if {[llength $argv]==2} {
set exe [lindex $argv 0]
set memfile [lindex $argv 1]
set report_at -1
} else {
set exe [lindex $argv 2]
set memfile [lindex $argv 3]
set report_at [lindex $argv 1]
}
proc process_input {input_file array_name} {
upvar $array_name mem
@ -68,6 +105,17 @@ proc process_input {input_file array_name} {
set mem($addr) [list $bytes "malloc $mallocid" $stack]
set stack ""
# Increase the current heap usage
incr ::nBytes $bytes
# Increase the number of malloc() calls
incr ::nMalloc
if {$::nBytes > $::nPeak} {
set ::nPeak $::nBytes
set ::iPeak $::nMalloc
}
} elseif { [regexp $FREE $line dummy bytes addr] } {
# If this is a 'free' line, remove the entry from the mem array. If the
# entry does not exist, or is the wrong number of bytes, announce a
@ -78,31 +126,86 @@ proc process_input {input_file array_name} {
}
unset mem($addr)
# Decrease the current heap usage
incr ::nBytes [expr -1 * $bytes]
} elseif { [regexp $REALLOC $line dummy mallocid ob b oa a] } {
# If it is a realloc line, remove the old mem entry and add a new one.
# "free" the old allocation in the internal model:
incr ::nBytes [expr -1 * $ob]
unset mem($oa);
# "malloc" the new allocation
set mem($a) [list $b "realloc $mallocid" $stack]
incr ::nBytes $b
set stack ""
# Increase the number of malloc() calls
incr ::nMalloc
if {$::nBytes > $::nPeak} {
set ::nPeak $::nBytes
set ::iPeak $::nMalloc
}
} else {
# puts "REJECT: $line"
}
if {$::nMalloc==$::report_at} report
}
close $input
}
process_input [lindex $argv 1] mem
set exe [lindex $argv 0]
foreach key [array names mem] {
set bytes [lindex $mem($key) 0]
set mallocid [lindex $mem($key) 1]
set stack [lindex $mem($key) 2]
puts "Leaked $bytes bytes at 0x$key: $mallocid"
foreach frame [lrange $stack 1 10] {
foreach {f l} [split [exec $addr2line -f --exe=$exe $frame] \n] {}
proc printstack {stack} {
set fcount 10
if {[llength $stack]<10} {
set fcount [llength $stack]
}
foreach frame [lrange $stack 1 $fcount] {
foreach {f l} [split [exec $::addr2line -f --exe=$::exe $frame] \n] {}
puts [format "%-30s %s" $f $l]
}
if {[llength $stack]>0 } {puts ""}
}
proc report {} {
foreach key [array names ::memmap] {
set stack [lindex $::memmap($key) 2]
set bytes [lindex $::memmap($key) 0]
lappend summarymap($stack) $bytes
}
foreach stack [array names summarymap] {
set allocs $summarymap($stack)
set sum 0
foreach a $allocs {
incr sum $a
}
lappend sorted [list $sum $stack]
}
set sorted [lsort -integer -index 0 $sorted]
foreach s $sorted {
set sum [lindex $s 0]
set stack [lindex $s 1]
set allocs $summarymap($stack)
puts "$sum bytes in [llength $allocs] chunks ($allocs)"
printstack $stack
}
# Print out summary statistics
puts "Total allocations : $::nMalloc"
puts "Total outstanding allocations: [array size ::memmap]"
puts "Current heap usage : $::nBytes bytes"
puts "Peak heap usage : $::nPeak bytes (malloc #$::iPeak)"
exit
}
process_input $memfile memmap
report