mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add some comments and test-cases for the global lru page list (used by sqlite3_release_memory()). (CVS 4308)

Browse Source 
FossilOrigin-Name: 0b80168895993af5774369f839f284712d006f0a
This commit is contained in:
danielk1977 2007-08-28 08:00:17 +00:00
parent 900b31ef49
commit 84f786fcdd
4 changed files with 194 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
manifest
View File

@ -1,5 +1,5 @@
C Work\stoward\scorrect\sbtree\slocking\sin\sa\smultithreaded\senvironment.\s(CVS\s4307)
D 2007-08-28T02:27:52
C Add\ssome\scomments\sand\stest-cases\sfor\sthe\sglobal\slru\spage\slist\s(used\sby\ssqlite3_release_memory()).\s(CVS\s4308)
D 2007-08-28T08:00:18
F Makefile.in e8296e112b8942a96c0ed504398bd0d43e3c67ce
F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@ -116,7 +116,7 @@ F src/os_unix.c 89bf24aa2475048a7833c45c522e7c6a81b83bb8
F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
F src/os_win.c 3ffd3aacff4cb69848284e29dcec0feff23b0752
F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
F src/pager.c cfa6dc38b797206549491de3ec7f0aea50611dda
F src/pager.c 51ca27639ab25c8838afc856d3cc6045a98752a7
F src/pager.h 53087c6fb9db01aed17c7fd044662a27507e89b8
F src/parse.y 2d2ce439dc6184621fb0b86f4fc5aca7f391a590
F src/pragma.c 9b989506a1b7c8aecd6befb8235e2f57a4aba7e5
@ -345,7 +345,7 @@ F test/malloc.test dbfaedfca734283182db18a64416cf037c33648f
F test/malloc2.test 96dfa6318d3b7bcd41d4fe7e24d35196d465bf58
F test/malloc3.test 65d323508c7c4e1fb5569d03a11070b0259ad2fe
F test/malloc4.test cf1e856fbc8ebde562838cb47adce0ebc2d4f7d2
F test/malloc5.test 6f7b96f1bea181d789a7140fd3fabfb0e333d8f5
F test/malloc5.test 9db9205a5d444e23043368f961f92641c7770199
F test/malloc6.test 3733b9bd4e039c3239f869c40edbb88172025e2e
F test/malloc7.test dd66d8f82916becf1d29b6640e4f4855485570f8
F test/malloc8.test 5ff95278bc73e815e295971afcdd175f6ba19258
@ -563,7 +563,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
P 741d6fb096dcb232871d3a8468c386022afcf554
R ff3e167d1bd072e0909067b1f2888076
U drh
Z 6d8ca04435be2f950a1ddef0045b914b
P b8cc493b47e618648f645ab73eb0253739e03fcd
R 89ac354b82d38fc45db7b69d7f01c4df
U danielk1977
Z 4eb080ba40501ea8bb2df84791a3ceaa

2
manifest.uuid
View File

@ -1 +1 @@
b8cc493b47e618648f645ab73eb0253739e03fcd
0b80168895993af5774369f839f284712d006f0a

78
src/pager.c
View File

@ -18,7 +18,7 @@
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.377 2007/08/27 17:27:49 danielk1977 Exp $
** @(#) $Id: pager.c,v 1.378 2007/08/28 08:00:18 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
@ -132,6 +132,33 @@
#define FORCE_ALIGNMENT(X) (((X)+7)&~7)
typedef struct PgHdr PgHdr;
/*
** Each pager stores all currently unreferenced pages in a list sorted
** in least-recently-used (LRU) order (i.e. the first item on the list has
** not been referenced in a long time, the last item has been recently
** used). An instance of this structure is included as part of each
** pager structure for this purpose (variable Pager.lru).
**
** Additionally, if memory-management is enabled, all unreferenced pages
** are stored in a global LRU list (global variable sqlite3LruPageList).
**
** In both cases, the PagerLruList.pFirstSynced variable points to
** the first page in the corresponding list that does not require an
** fsync() operation before it's memory can be reclaimed. If no such
** page exists, PagerLruList.pFirstSynced is set to NULL.
*/
typedef struct PagerLruList PagerLruList;
struct PagerLruList {
PgHdr *pFirst; /* First page in LRU list */
PgHdr *pLast; /* Last page in LRU list (the most recently used) */
PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */
};
/*
** The following structure contains the next and previous pointers used
** to link a PgHdr structure into a PagerLruList linked list.
*/
typedef struct PagerLruLink PagerLruLink;
struct PagerLruLink {
PgHdr *pNext;
@ -292,13 +319,6 @@ struct PgHistory {
#define PGHDR_TO_HIST(P,PGR) \
((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra])
typedef struct PagerLruList PagerLruList;
struct PagerLruList {
PgHdr *pFirst;
PgHdr *pLast;
PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */
};
/*
** A open page cache is an instance of the following structure.
**
@ -531,10 +551,22 @@ static const unsigned char aJournalMagic[] = {
# define REFINFO(X)
#endif
/*
** Add page pPg to the end of the linked list managed by structure
** pList (pPg becomes the last entry in the list - the most recently
** used). Argument pLink should point to either pPg->free or pPg->gfree,
** depending on whether pPg is being added to the pager-specific or
** global LRU list.
*/
static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
pLink->pNext = 0;
pLink->pPrev = pList->pLast;
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
assert(pLink==&pPg->free || pLink==&pPg->gfree);
assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList);
#endif
if( pList->pLast ){
int iOff = (char *)pLink - (char *)pPg;
PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]);
@ -550,9 +582,20 @@ static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
}
}
/*
** Remove pPg from the list managed by the structure pointed to by pList.
**
** Argument pLink should point to either pPg->free or pPg->gfree, depending
** on whether pPg is being added to the pager-specific or global LRU list.
*/
static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
int iOff = (char *)pLink - (char *)pPg;
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
assert(pLink==&pPg->free || pLink==&pPg->gfree);
assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList);
#endif
if( pPg==pList->pFirst ){
pList->pFirst = pLink->pNext;
}
@ -580,7 +623,9 @@ static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){
}
/*
** Add page to the free-list
** Add page pPg to the list of free pages for the pager. If
** memory-management is enabled, also add the page to the global
** list of free pages.
*/
static void lruListAdd(PgHdr *pPg){
listAdd(&pPg->pPager->lru, &pPg->free, pPg);
@ -594,7 +639,9 @@ static void lruListAdd(PgHdr *pPg){
}
/*
** Remove page from free-list
** Remove page pPg from the list of free pages for the associated pager.
** If memory-management is enabled, also remove pPg from the global list
** of free pages.
*/
static void lruListRemove(PgHdr *pPg){
listRemove(&pPg->pPager->lru, &pPg->free, pPg);
@ -608,7 +655,11 @@ static void lruListRemove(PgHdr *pPg){
}
/*
** Set the Pager.pFirstSynced variable
** This function is called just after the needSync flag has been cleared
** from all pages managed by pPager (usually because the journal file
** has just been synced). It updates the pPager->lru.pFirstSynced variable
** and, if memory-management is enabled, the sqlite3LruPageList.pFirstSynced
** variable also.
*/
static void lruListSetFirstSynced(Pager *pPager){
pPager->lru.pFirstSynced = pPager->lru.pFirst;
@ -3056,6 +3107,10 @@ int sqlite3PagerReleaseMemory(int nReq){
}
sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
/* If pPg==0, then the block above has failed to find a page to
** recycle. In this case return early - no further memory will
** be released.
*/
if( !pPg ) break;
pPager = pPg->pPager;
@ -3089,6 +3144,7 @@ int sqlite3PagerReleaseMemory(int nReq){
IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno));
PAGER_INCR(sqlite3_pager_pgfree_count);
sqlite3_free(pPg);
pPager->nPage--;
}else{
/* An error occured whilst writing to the database file or
** journal in pager_recycle(). The error is not returned to the

120
test/malloc5.test
View File

@ -12,7 +12,7 @@
# This file contains test cases focused on the two memory-management APIs,
# sqlite3_soft_heap_limit() and sqlite3_release_memory().
#
# $Id: malloc5.test,v 1.13 2007/08/22 22:04:37 drh Exp $
# $Id: malloc5.test,v 1.14 2007/08/28 08:00:18 danielk1977 Exp $
#---------------------------------------------------------------------------
# NOTES ON EXPECTED BEHAVIOUR
@ -248,7 +248,7 @@ do_test malloc5-5.1 {
}
sqlite3_release_memory
} 0
do_test malloc5-5.1 {
do_test malloc5-5.2 {
sqlite3_soft_heap_limit 5000
execsql {
COMMIT;
@ -257,6 +257,122 @@ do_test malloc5-5.1 {
}
expr 1
} {1}
sqlite3_soft_heap_limit $::soft_limit
#-------------------------------------------------------------------------
# The following test cases (malloc5-6.*) test the new global LRU list
# used to determine the pages to recycle when sqlite3_release_memory is
# called and there is more than one pager open.
#
proc nPage {db} {
set bt [btree_from_db $db]
array set stats [btree_pager_stats $bt]
set stats(page)
}
db close
file delete -force test.db test.db-journal test2.db test2.db-journal
# This block of test-cases (malloc5-6.1.*) prepares two database files
# for the subsequent tests.
do_test malloc5-6.1.1 {
sqlite3 db test.db
execsql {
PRAGMA page_size=1024;
PRAGMA default_cache_size=10;
BEGIN;
CREATE TABLE abc(a PRIMARY KEY, b, c);
INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
COMMIT;
}
copy_file test.db test2.db
sqlite3 db2 test2.db
list [expr [file size test.db]/1024] [expr [file size test2.db]/1024]
} {23 23}
do_test malloc5-6.1.2 {
list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
} {10 10}
do_test malloc5-6.2.1 {
execsql { SELECT * FROM abc } db2
execsql {SELECT * FROM abc} db
list [nPage db] [nPage db2]
} {10 10}
do_test malloc5-6.2.2 {
# If we now try to reclaim some memory, it should come from the db2 cache.
sqlite3_release_memory 3000
list [nPage db] [nPage db2]
} {10 7}
do_test malloc5-6.2.3 {
# Access the db2 cache again, so that all the db2 pages have been used
# more recently than all the db pages. Then try to reclaim 3000 bytes.
# This time, 3 pages should be pulled from the db cache.
execsql { SELECT * FROM abc } db2
sqlite3_release_memory 3000
list [nPage db] [nPage db2]
} {7 10}
do_test malloc5-6.3.1 {
# Now open a transaction and update 2 pages in the db2 cache. Then
# do a SELECT on the db cache so that all the db pages are more recently
# used than the db2 pages. When we try to free memory, SQLite should
# free the non-dirty db2 pages, then the db pages, then finally use
# sync() to free up the dirty db2 pages. The only page that cannot be
# freed is page1 of db2. Because there is an open transaction, the
# btree layer holds a reference to page 1 in the db2 cache.
execsql {
BEGIN;
UPDATE abc SET c = randstr(100,100)
WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);
} db2
execsql { SELECT * FROM abc } db
list [nPage db] [nPage db2]
} {10 10}
do_test malloc5-6.3.2 {
# Try to release 7700 bytes. This should release all the
# non-dirty pages held by db2.
sqlite3_release_memory [expr 7*1100]
list [nPage db] [nPage db2]
} {10 3}
do_test malloc5-6.3.3 {
# Try to release another 1000 bytes. This should come fromt the db
# cache, since all three pages held by db2 are either in-use or diry.
sqlite3_release_memory 1000
list [nPage db] [nPage db2]
} {9 3}
do_test malloc5-6.3.4 {
# Now release 9900 more (about 9 pages worth). This should expunge
# the rest of the db cache. But the db2 cache remains intact, because
# SQLite tries to avoid calling sync().
sqlite3_release_memory 9900
list [nPage db] [nPage db2]
} {0 3}
do_test malloc5-6.3.5 {
# But if we are really insistent, SQLite will consent to call sync()
# if there is no other option.
sqlite3_release_memory 1000
list [nPage db] [nPage db2]
} {0 2}
do_test malloc5-6.3.6 {
# The referenced page (page 1 of the db2 cache) will not be freed no
# matter how much memory we ask for:
sqlite3_release_memory 31459
list [nPage db] [nPage db2]
} {0 1}
db2 close
sqlite3_soft_heap_limit $::soft_limit
finish_test