Defer the bulk pcache1 memory allocation until the first page allocation

request.  Limit the size of the pcache1 bulk allocation to the cache_size
setting.  Deallocate the bulk allocation on a sqlite3_db_release_memory()
request, if the bulk allocation is completely unused.

FossilOrigin-Name: b79a4affe44bd0c8e155cae19f3f62c715684cd6
This commit is contained in:
drh 2015-07-16 18:18:19 +00:00
parent 89b31d73a6
commit 957026ac78
9 changed files with 91 additions and 68 deletions

View File

@ -1,5 +1,5 @@
C Fix\scompilation\sissues\swith\sSQLITE_OMIT_COMPOUND_SELECT\sdefined.
D 2015-07-16T17:29:27.953
C Defer\sthe\sbulk\spcache1\smemory\sallocation\suntil\sthe\sfirst\spage\sallocation\nrequest.\s\sLimit\sthe\ssize\sof\sthe\spcache1\sbulk\sallocation\sto\sthe\scache_size\nsetting.\s\sDeallocate\sthe\sbulk\sallocation\son\sa\ssqlite3_db_release_memory()\nrequest,\sif\sthe\sbulk\sallocation\sis\scompletely\sunused.
D 2015-07-16T18:18:19.580
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 6e8af213d49e6325bf283ebed7662254f8e15bda
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -319,9 +319,9 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8
F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0
F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
F src/pcache1.c 3f4c87cf913f2de8189026d9a5223ddaf55eaf6b
F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7
F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc
F src/pcache1.c 6c1e5957087b74e7e7cff871561849cf2fe09654
F src/pragma.c e52084b37a08a88f258830518461e94627af2621
F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a
F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc
F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
@ -823,7 +823,7 @@ F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a
F test/malloc4.test 957337613002b7058a85116493a262f679f3a261
F test/malloc5.test e9a9116f80ab6b7a9ca258876c6f3dedb08cb08b
F test/malloc5.test 21f6552bacb9efe649e6ba10a52f2cedaecce242
F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151
F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a
F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d
@ -900,8 +900,8 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6
F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305
F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
F test/pcache2.test ec3ae192f444ee6a0a80d1fd80d99695d884bfb3
F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
F test/pcache2.test c70d92547550136ba6f818e6a44fe246d2738604
F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
F test/permutations.test 6a88fd9ca15b804e9c20990773262ca67494058f
F test/pragma.test be7195f0aa72bdb8a512133e9640ac40f15b57a2
@ -1327,7 +1327,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f
F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
F tool/mkpragmatab.tcl 40c287d3f929ece67da6e9e7c49885789960accf
F tool/mkpragmatab.tcl 84af2b180484323a2ea22a2279e8bd9e3e1e492e
F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
F tool/mksqlite3c-noext.tcl 87240b09c20042999b41d5fabe091b7111287835
F tool/mksqlite3c.tcl f29898d34f1dcd77ccc4e6fd7dcc2f9ebb54622f
@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 2cdd647951ff5dca53576bb8be6dd6310a557571
R 2352a36372e4c50dccbac9452cf765fb
U mistachkin
Z ddf00ea17d7e53d080e6ac931c2ced93
P 9c39d4644530ccc532f4ff26464106c6da43269a
R 09036069f4d0dade4c9d3cbed758be18
U drh
Z 38e6769945da8f33e2e2b46c42a73ce6

View File

@ -1 +1 @@
9c39d4644530ccc532f4ff26464106c6da43269a
b79a4affe44bd0c8e155cae19f3f62c715684cd6

View File

@ -191,6 +191,7 @@ static SQLITE_WSD struct PCacheGlobal {
*/
int isInit; /* True if initialized */
int separateCache; /* Use a new PGroup for each PCache */
int nInitPage; /* Initial bulk allocation size */
int szSlot; /* Size of each free slot */
int nSlot; /* The number of pcache slots */
int nReserve; /* Try to keep nFreeSlot above this */
@ -259,6 +260,43 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
}
}
/*
** Try to initialize the pCache->pFree and pCache->pBulk fields. Return
** true if pCache->pFree ends up containing one or more free pages.
*/
static int pcache1InitBulk(PCache1 *pCache){
int szBulk;
char *zBulk;
if( pcache1.nInitPage==0 ) return 0;
/* Do not bother with a bulk allocation if the cache size very small */
if( pCache->nMax<3 ) return 0;
sqlite3BeginBenignMalloc();
if( pcache1.nInitPage>0 ){
szBulk = pCache->szAlloc * pcache1.nInitPage;
}else{
szBulk = -1024*pcache1.nInitPage;
}
if( szBulk > pCache->szAlloc*pCache->nMax ){
szBulk = pCache->szAlloc*pCache->nMax;
}
zBulk = pCache->pBulk = sqlite3Malloc( szBulk );
sqlite3EndBenignMalloc();
if( zBulk ){
int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc;
int i;
for(i=0; i<nBulk; i++){
PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage];
pX->page.pBuf = zBulk;
pX->page.pExtra = &pX[1];
pX->isBulkLocal = 1;
pX->pNext = pCache->pFree;
pCache->pFree = pX;
zBulk += pCache->szAlloc;
}
}
return pCache->pFree!=0;
}
/*
** Malloc function used within this file to allocate space from the buffer
** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no
@ -359,7 +397,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
void *pPg;
assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
if( pCache->pFree ){
if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){
p = pCache->pFree;
pCache->pFree = p->pNext;
p->pNext = 0;
@ -563,7 +601,8 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){
** If there are currently more than nMaxPage pages allocated, try
** to recycle pages to reduce the number allocated to nMaxPage.
*/
static void pcache1EnforceMaxPage(PGroup *pGroup){
static void pcache1EnforceMaxPage(PCache1 *pCache){
PGroup *pGroup = pCache->pGroup;
assert( sqlite3_mutex_held(pGroup->mutex) );
while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){
PgHdr1 *p = pGroup->pLruTail;
@ -572,6 +611,10 @@ static void pcache1EnforceMaxPage(PGroup *pGroup){
pcache1PinPage(p);
pcache1RemoveFromHash(p, 1);
}
if( pCache->nPage==0 && pCache->pBulk ){
sqlite3_free(pCache->pBulk);
pCache->pBulk = pCache->pFree = 0;
}
}
/*
@ -647,6 +690,14 @@ static int pcache1Init(void *NotUsed){
pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM);
}
#endif
if( pcache1.separateCache
&& sqlite3GlobalConfig.nPage!=0
&& sqlite3GlobalConfig.pPage==0
){
pcache1.nInitPage = sqlite3GlobalConfig.nPage;
}else{
pcache1.nInitPage = 0;
}
pcache1.grp.mxPinned = 10;
pcache1.isInit = 1;
return SQLITE_OK;
@ -701,36 +752,6 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
}
pcache1LeaveMutex(pGroup);
/* Try to initialize the local bulk pagecache line allocation if using
** separate caches and if nPage!=0 */
if( pcache1.separateCache
&& sqlite3GlobalConfig.nPage!=0
&& sqlite3GlobalConfig.pPage==0
){
int szBulk;
char *zBulk;
sqlite3BeginBenignMalloc();
if( sqlite3GlobalConfig.nPage>0 ){
szBulk = pCache->szAlloc * sqlite3GlobalConfig.nPage;
}else{
szBulk = -1024*sqlite3GlobalConfig.nPage;
}
zBulk = pCache->pBulk = sqlite3Malloc( szBulk );
sqlite3EndBenignMalloc();
if( zBulk ){
int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc;
int i;
for(i=0; i<nBulk; i++){
PgHdr1 *pX = (PgHdr1*)&zBulk[szPage];
pX->page.pBuf = zBulk;
pX->page.pExtra = &pX[1];
pX->isBulkLocal = 1;
pX->pNext = pCache->pFree;
pCache->pFree = pX;
zBulk += pCache->szAlloc;
}
}
}
if( pCache->nHash==0 ){
pcache1Destroy((sqlite3_pcache*)pCache);
pCache = 0;
@ -753,7 +774,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){
pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
pCache->nMax = nMax;
pCache->n90pct = pCache->nMax*9/10;
pcache1EnforceMaxPage(pGroup);
pcache1EnforceMaxPage(pCache);
pcache1LeaveMutex(pGroup);
}
}
@ -771,7 +792,7 @@ static void pcache1Shrink(sqlite3_pcache *p){
pcache1EnterMutex(pGroup);
savedMaxPage = pGroup->nMaxPage;
pGroup->nMaxPage = 0;
pcache1EnforceMaxPage(pGroup);
pcache1EnforceMaxPage(pCache);
pGroup->nMaxPage = savedMaxPage;
pcache1LeaveMutex(pGroup);
}
@ -1108,7 +1129,7 @@ static void pcache1Destroy(sqlite3_pcache *p){
assert( pGroup->nMinPage >= pCache->nMin );
pGroup->nMinPage -= pCache->nMin;
pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
pcache1EnforceMaxPage(pGroup);
pcache1EnforceMaxPage(pCache);
pcache1LeaveMutex(pGroup);
sqlite3_free(pCache->pBulk);
sqlite3_free(pCache->apHash);

View File

@ -721,6 +721,7 @@ void sqlite3Pragma(
case PragTyp_CACHE_SIZE: {
assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
if( !zRight ){
if( sqlite3ReadSchema(pParse) ) goto pragma_out;
returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size);
}else{
int size = sqlite3Atoi(zRight);

View File

@ -86,7 +86,7 @@ static const struct sPragmaNames {
#if !defined(SQLITE_OMIT_PAGER_PRAGMAS)
{ /* zName: */ "cache_size",
/* ePragTyp: */ PragTyp_CACHE_SIZE,
/* ePragFlag: */ PragFlag_NeedSchema,
/* ePragFlag: */ 0,
/* iArg: */ 0 },
#endif
#if !defined(SQLITE_OMIT_FLAG_PRAGMAS)

View File

@ -41,6 +41,7 @@ ifcapable !memorymanage {
sqlite3_soft_heap_limit 0
sqlite3 db test.db
db eval {PRAGMA cache_size=1}
do_test malloc5-1.1 {
# Simplest possible test. Call sqlite3_release_memory when there is exactly
@ -61,15 +62,14 @@ do_test malloc5-1.2 {
# not the case before version 3.6.2).
#
sqlite3 db2 test.db
execsql { SELECT * FROM sqlite_master } db2
execsql {PRAGMA cache_size=2; SELECT * FROM sqlite_master } db2
} {}
do_test malloc5-1.3 {
# Call [sqlite3_release_memory] when there is exactly one unused page
# in the cache belonging to db2.
#
set ::pgalloc [sqlite3_release_memory]
expr $::pgalloc > 0
} {1}
} {0}
# The sizes of memory allocations from system malloc() might vary,
# depending on the memory allocator algorithms used. The following
@ -231,6 +231,7 @@ do_test malloc5-4.1 {
expr $nMaxBytes > 1000000
} {1}
do_test malloc5-4.2 {
db eval {PRAGMA cache_size=1}
db cache flush
sqlite3_release_memory
sqlite3_soft_heap_limit 100000
@ -302,7 +303,7 @@ do_test malloc5-6.1.1 {
sqlite3 db test.db
execsql {
PRAGMA page_size=1024;
PRAGMA default_cache_size=10;
PRAGMA default_cache_size=2;
}
execsql {
PRAGMA temp_store = memory;
@ -325,24 +326,25 @@ do_test malloc5-6.1.1 {
}
forcecopy test.db test2.db
sqlite3 db2 test2.db
db2 eval {PRAGMA cache_size=2}
list \
[expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]
} {1 1}
do_test malloc5-6.1.2 {
list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
} {10 10}
} {2 2}
do_test malloc5-6.2.1 {
execsql {SELECT * FROM abc} db2
execsql {SELECT * FROM abc} db
expr [nPage db] + [nPage db2]
} {20}
} {4}
do_test malloc5-6.2.2 {
# If we now try to reclaim some memory, it should come from the db2 cache.
sqlite3_release_memory 3000
expr [nPage db] + [nPage db2]
} {17}
} {4}
do_test malloc5-6.2.3 {
# Access the db2 cache again, so that all the db2 pages have been used
# more recently than all the db pages. Then try to reclaim 3000 bytes.
@ -350,7 +352,7 @@ do_test malloc5-6.2.3 {
execsql { SELECT * FROM abc } db2
sqlite3_release_memory 3000
expr [nPage db] + [nPage db2]
} {17}
} {4}
do_test malloc5-6.3.1 {
# Now open a transaction and update 2 pages in the db2 cache. Then
@ -367,19 +369,19 @@ do_test malloc5-6.3.1 {
} db2
execsql { SELECT * FROM abc } db
expr [nPage db] + [nPage db2]
} {20}
} {4}
do_test malloc5-6.3.2 {
# Try to release 7700 bytes. This should release all the
# non-dirty pages held by db2.
sqlite3_release_memory [expr 7*1132]
list [nPage db] [nPage db2]
} {10 3}
} {1 3}
do_test malloc5-6.3.3 {
# Try to release another 1000 bytes. This should come fromt the db
# cache, since all three pages held by db2 are either in-use or diry.
sqlite3_release_memory 1000
list [nPage db] [nPage db2]
} {9 3}
} {1 3}
do_test malloc5-6.3.4 {
# Now release 9900 more (about 9 pages worth). This should expunge
# the rest of the db cache. But the db2 cache remains intact, because
@ -390,20 +392,20 @@ do_test malloc5-6.3.4 {
sqlite3_release_memory 9900
}
list [nPage db] [nPage db2]
} {0 3}
} {1 3}
do_test malloc5-6.3.5 {
# But if we are really insistent, SQLite will consent to call sync()
# if there is no other option. UPDATE: As of 3.6.2, SQLite will not
# call sync() in this scenario. So no further memory can be reclaimed.
sqlite3_release_memory 1000
list [nPage db] [nPage db2]
} {0 3}
} {1 3}
do_test malloc5-6.3.6 {
# The referenced page (page 1 of the db2 cache) will not be freed no
# matter how much memory we ask for:
sqlite3_release_memory 31459
list [nPage db] [nPage db2]
} {0 3}
} {1 3}
db2 close

View File

@ -73,7 +73,7 @@ do_test pcache-1.4 {
do_test pcache-1.5 {
sqlite3 db2 test.db
execsql "PRAGMA cache_size=10" db2
execsql "PRAGMA cache_size; PRAGMA cache_size=10" db2
pcache_stats
} {current 11 max 22 min 20 recyclable 1}

View File

@ -36,13 +36,13 @@ do_test pcache2-1.1 {
do_test pcache2-1.2 {
forcedelete test.db test.db-journal
sqlite3 db test.db
db eval {PRAGMA cache_size=10}
db eval {PRAGMA cache_size=10; SELECT 1 FROM sqlite_master;}
lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 1
} {2}
do_test pcache2-1.3 {
forcedelete test2.db test2.db-journal
sqlite3 db2 test2.db
db2 eval {PRAGMA cache_size=50}
db2 eval {PRAGMA cache_size=50; SELECT 1 FROM sqlite_master;}
lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 1
} {4}

View File

@ -170,7 +170,6 @@ set pragma_def {
IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS)
NAME: cache_size
FLAG: NeedSchema
IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS)
NAME: mmap_size