From ff59ce680df5d2032ea5a11c666688269225f033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Axel=20D=C3=B6rfler?= Date: Wed, 24 Feb 2010 14:43:20 +0000 Subject: [PATCH] * The low resource handler now empties the cache depot's magazines; before, they were never freed unless the cache was destroyed (I just wondered why my system would bury >1G in the magazines). * Made the magazine capacity variable per cache, ie. for larger objects, it's not a good idea to have 64*CPU buffers lying around in the worst case. * Furthermore, the create_object_cache_etc()/object_depot_init() now have arguments for the magazine capacity as well as the maximum number of full unused magazines. * By default, you might want to initialize both to zero, as then some hopefully usable defaults are computed. Otherwise (the only current example is the vm_page_mapping cache) you can just put in the values you'd want there. The page mapping cache uses larger values, as its objects are usually allocated and deleted in larger chunks. * Beware, though, I couldn't test these changes yet as Qemu didn't like to run today. I'll test these changes on another machine now. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@35601 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/slab/ObjectDepot.h | 9 ++- headers/private/kernel/slab/Slab.h | 11 ++-- src/system/kernel/cache/block_cache.cpp | 10 ++-- src/system/kernel/slab/HashedObjectCache.cpp | 8 ++- src/system/kernel/slab/HashedObjectCache.h | 2 + src/system/kernel/slab/ObjectCache.cpp | 24 +++++--- src/system/kernel/slab/ObjectCache.h | 6 +- src/system/kernel/slab/ObjectDepot.cpp | 58 ++++++++++++++------ src/system/kernel/slab/Slab.cpp | 23 +++++--- src/system/kernel/slab/SmallObjectCache.cpp | 8 ++- src/system/kernel/slab/SmallObjectCache.h | 2 + src/system/kernel/slab/allocator.cpp | 4 +- src/system/kernel/vm/vm.cpp | 6 +- 13 files changed, 112 insertions(+), 59 deletions(-) diff --git a/headers/private/kernel/slab/ObjectDepot.h b/headers/private/kernel/slab/ObjectDepot.h index 47b2ee5f03..9c2fe557a9 100644 --- a/headers/private/kernel/slab/ObjectDepot.h +++ b/headers/private/kernel/slab/ObjectDepot.h @@ -1,4 +1,5 @@ /* + * Copyright 2010, Axel Dörfler. All Rights Reserved. * Copyright 2007, Hugo Santos. All Rights Reserved. * Distributed under the terms of the MIT License. */ @@ -19,9 +20,11 @@ typedef struct object_depot { DepotMagazine* empty; size_t full_count; size_t empty_count; + size_t max_count; + size_t magazine_capacity; struct depot_cpu_store* stores; + void* cookie; - void* cookie; void (*return_object)(struct object_depot* depot, void* cookie, void* object, uint32 flags); } object_depot; @@ -31,7 +34,8 @@ typedef struct object_depot { extern "C" { #endif -status_t object_depot_init(object_depot* depot, uint32 flags, void* cookie, +status_t object_depot_init(object_depot* depot, size_t capacity, + size_t maxCount, uint32 flags, void* cookie, void (*returnObject)(object_depot* depot, void* cookie, void* object, uint32 flags)); void object_depot_destroy(object_depot* depot, uint32 flags); @@ -45,4 +49,5 @@ void object_depot_make_empty(object_depot* depot, uint32 flags); } #endif + #endif /* _SLAB_OBJECT_DEPOT_H_ */ diff --git a/headers/private/kernel/slab/Slab.h b/headers/private/kernel/slab/Slab.h index a0b1bd2d22..a3eb79506a 100644 --- a/headers/private/kernel/slab/Slab.h +++ b/headers/private/kernel/slab/Slab.h @@ -1,5 +1,5 @@ /* - * Copyright 2008, Axel Dörfler. All Rights Reserved. + * Copyright 2008-2010, Axel Dörfler. All Rights Reserved. * Copyright 2007, Hugo Santos. All Rights Reserved. * * Distributed under the terms of the MIT License. @@ -42,11 +42,12 @@ typedef void (*object_cache_reclaimer)(void* cookie, int32 level); extern "C" { #endif -object_cache* create_object_cache(const char* name, size_t object_size, +object_cache* create_object_cache(const char* name, size_t objectSize, size_t alignment, void* cookie, object_cache_constructor constructor, - object_cache_destructor); -object_cache* create_object_cache_etc(const char* name, size_t object_size, - size_t alignment, size_t max_byte_usage, uint32 flags, void* cookie, + object_cache_destructor destructor); +object_cache* create_object_cache_etc(const char* name, size_t objectSize, + size_t alignment, size_t maxByteUsage, size_t magazineCapacity, + size_t maxMagazineCount, uint32 flags, void* cookie, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer); diff --git a/src/system/kernel/cache/block_cache.cpp b/src/system/kernel/cache/block_cache.cpp index dd8ab4e0ec..d77c4cf4ff 100644 --- a/src/system/kernel/cache/block_cache.cpp +++ b/src/system/kernel/cache/block_cache.cpp @@ -1329,7 +1329,7 @@ block_cache::Init() mutex_init(&lock, "block cache"); buffer_cache = create_object_cache_etc("block cache buffers", block_size, - 8, 0, CACHE_LARGE_SLAB, NULL, NULL, NULL, NULL); + 8, 0, 0, 0, CACHE_LARGE_SLAB, NULL, NULL, NULL, NULL); if (buffer_cache == NULL) return B_NO_MEMORY; @@ -1472,7 +1472,7 @@ block_cache::RemoveUnusedBlocks(int32 maxAccessed, int32 count) if (block->is_dirty && !block->discard) { if (block->busy_writing) continue; - + BlockWriter::WriteBlock(this, block); } @@ -2022,7 +2022,7 @@ dump_block(cached_block* block) (addr_t)block, block->block_number, (addr_t)block->current_data, (addr_t)block->original_data, (addr_t)block->parent_data, block->ref_count, block->accessed, - block->busy_reading ? 'r' : '-', block->busy_writing ? 'w' : '-', + block->busy_reading ? 'r' : '-', block->busy_writing ? 'w' : '-', block->is_writing ? 'W' : '-', block->is_dirty ? 'D' : '-', block->unused ? 'U' : '-', block->discard ? 'D' : '-', (addr_t)block->transaction, @@ -2587,7 +2587,7 @@ status_t block_cache_init(void) { sBlockCache = create_object_cache_etc("cached blocks", sizeof(cached_block), - 8, 0, CACHE_LARGE_SLAB, NULL, NULL, NULL, NULL); + 8, 0, 0, 0, CACHE_LARGE_SLAB, NULL, NULL, NULL, NULL); if (sBlockCache == NULL) return B_NO_MEMORY; @@ -3299,7 +3299,7 @@ block_cache_sync(void* _cache) } hash_close(cache->hash, &iterator, false); - + status_t status = writer.Write(); locker.Unlock(); diff --git a/src/system/kernel/slab/HashedObjectCache.cpp b/src/system/kernel/slab/HashedObjectCache.cpp index cab4472972..5d2bc68a56 100644 --- a/src/system/kernel/slab/HashedObjectCache.cpp +++ b/src/system/kernel/slab/HashedObjectCache.cpp @@ -51,7 +51,8 @@ HashedObjectCache::HashedObjectCache() /*static*/ HashedObjectCache* HashedObjectCache::Create(const char* name, size_t object_size, - size_t alignment, size_t maximum, uint32 flags, void* cookie, + size_t alignment, size_t maximum, size_t magazineCapacity, + size_t maxMagazineCount, uint32 flags, void* cookie, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer) { @@ -71,8 +72,9 @@ HashedObjectCache::Create(const char* name, size_t object_size, cache->hash_table.Resize(buffer, hashSize, true); - if (cache->Init(name, object_size, alignment, maximum, flags, cookie, - constructor, destructor, reclaimer) != B_OK) { + if (cache->Init(name, object_size, alignment, maximum, magazineCapacity, + maxMagazineCount, flags, cookie, constructor, destructor, + reclaimer) != B_OK) { cache->Delete(); return NULL; } diff --git a/src/system/kernel/slab/HashedObjectCache.h b/src/system/kernel/slab/HashedObjectCache.h index 00c0acbdc6..7007ad68d7 100644 --- a/src/system/kernel/slab/HashedObjectCache.h +++ b/src/system/kernel/slab/HashedObjectCache.h @@ -24,6 +24,8 @@ struct HashedObjectCache : ObjectCache { static HashedObjectCache* Create(const char* name, size_t object_size, size_t alignment, size_t maximum, + size_t magazineCapacity, + size_t maxMagazineCount, uint32 flags, void* cookie, object_cache_constructor constructor, object_cache_destructor destructor, diff --git a/src/system/kernel/slab/ObjectCache.cpp b/src/system/kernel/slab/ObjectCache.cpp index 88ce1f51e4..de6cbb5caf 100644 --- a/src/system/kernel/slab/ObjectCache.cpp +++ b/src/system/kernel/slab/ObjectCache.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2008, Axel Dörfler. All Rights Reserved. + * Copyright 2008-2010, Axel Dörfler. All Rights Reserved. * Copyright 2007, Hugo Santos. All Rights Reserved. * * Distributed under the terms of the MIT License. @@ -40,10 +40,10 @@ ObjectCache::~ObjectCache() status_t -ObjectCache::Init(const char* name, size_t objectSize, - size_t alignment, size_t maximum, uint32 flags, void* cookie, - object_cache_constructor constructor, object_cache_destructor destructor, - object_cache_reclaimer reclaimer) +ObjectCache::Init(const char* name, size_t objectSize, size_t alignment, + size_t maximum, size_t magazineCapacity, size_t maxMagazineCount, + uint32 flags, void* cookie, object_cache_constructor constructor, + object_cache_destructor destructor, object_cache_reclaimer reclaimer) { strlcpy(this->name, name, sizeof(this->name)); @@ -86,9 +86,17 @@ ObjectCache::Init(const char* name, size_t objectSize, this->flags |= CACHE_NO_DEPOT; if (!(this->flags & CACHE_NO_DEPOT)) { - status_t status = object_depot_init(&depot, flags, this, - object_cache_return_object_wrapper); - if (status < B_OK) { + // Determine usable magazine configuration values if none had been given + if (magazineCapacity == 0) { + magazineCapacity = objectSize < 256 + ? 32 : (objectSize < 512 ? 16 : 8); + } + if (maxMagazineCount == 0) + maxMagazineCount = magazineCapacity / 2; + + status_t status = object_depot_init(&depot, magazineCapacity, + maxMagazineCount, flags, this, object_cache_return_object_wrapper); + if (status != B_OK) { mutex_destroy(&lock); return status; } diff --git a/src/system/kernel/slab/ObjectCache.h b/src/system/kernel/slab/ObjectCache.h index febc36b865..3ee13cbe26 100644 --- a/src/system/kernel/slab/ObjectCache.h +++ b/src/system/kernel/slab/ObjectCache.h @@ -1,5 +1,5 @@ /* - * Copyright 2008, Axel Dörfler. All Rights Reserved. + * Copyright 2008-2010, Axel Dörfler. All Rights Reserved. * Copyright 2007, Hugo Santos. All Rights Reserved. * * Distributed under the terms of the MIT License. @@ -80,7 +80,9 @@ public: status_t Init(const char* name, size_t objectSize, size_t alignment, size_t maximum, - uint32 flags, void* cookie, + size_t magazineCapacity, + size_t maxMagazineCount, uint32 flags, + void* cookie, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer); diff --git a/src/system/kernel/slab/ObjectDepot.cpp b/src/system/kernel/slab/ObjectDepot.cpp index 4f219ece24..47b6dcc190 100644 --- a/src/system/kernel/slab/ObjectDepot.cpp +++ b/src/system/kernel/slab/ObjectDepot.cpp @@ -19,10 +19,6 @@ #include "slab_private.h" -static const int kMagazineCapacity = 32; - // TODO: Should be dynamically tuned per cache. - - struct DepotMagazine { DepotMagazine* next; uint16 current_round; @@ -80,14 +76,15 @@ DepotMagazine::Push(void* object) static DepotMagazine* -alloc_magazine(uint32 flags) +alloc_magazine(object_depot* depot, uint32 flags) { DepotMagazine* magazine = (DepotMagazine*)slab_internal_alloc( - sizeof(DepotMagazine) + kMagazineCapacity * sizeof(void*), flags); + sizeof(DepotMagazine) + depot->magazine_capacity * sizeof(void*), + flags); if (magazine) { magazine->next = NULL; magazine->current_round = 0; - magazine->round_count = kMagazineCapacity; + magazine->round_count = depot->magazine_capacity; } return magazine; @@ -113,6 +110,8 @@ empty_magazine(object_depot* depot, DepotMagazine* magazine, uint32 flags) static bool exchange_with_full(object_depot* depot, DepotMagazine*& magazine) { + ASSERT(magazine->IsEmpty()); + SpinLocker _(depot->inner_lock); if (depot->full == NULL) @@ -128,8 +127,11 @@ exchange_with_full(object_depot* depot, DepotMagazine*& magazine) static bool -exchange_with_empty(object_depot* depot, DepotMagazine*& magazine) +exchange_with_empty(object_depot* depot, DepotMagazine*& magazine, + DepotMagazine*& freeMagazine) { + ASSERT(magazine == NULL || magazine->IsFull()); + SpinLocker _(depot->inner_lock); if (depot->empty == NULL) @@ -137,9 +139,12 @@ exchange_with_empty(object_depot* depot, DepotMagazine*& magazine) depot->empty_count--; - if (magazine) { - _push(depot->full, magazine); - depot->full_count++; + if (magazine != NULL) { + if (depot->full_count < depot->max_count) { + _push(depot->full, magazine); + depot->full_count++; + } else + freeMagazine = magazine; } magazine = _pop(depot->empty); @@ -168,13 +173,15 @@ object_depot_cpu(object_depot* depot) status_t -object_depot_init(object_depot* depot, uint32 flags, void* cookie, - void (*return_object)(object_depot* depot, void* cookie, void* object, - uint32 flags)) +object_depot_init(object_depot* depot, size_t capacity, size_t maxCount, + uint32 flags, void* cookie, void (*return_object)(object_depot* depot, + void* cookie, void* object, uint32 flags)) { depot->full = NULL; depot->empty = NULL; depot->full_count = depot->empty_count = 0; + depot->max_count = maxCount; + depot->magazine_capacity = capacity; rw_lock_init(&depot->outer_lock, "object depot"); B_INITIALIZE_SPINLOCK(&depot->inner_lock); @@ -245,6 +252,8 @@ object_depot_obtain(object_depot* depot) int object_depot_store(object_depot* depot, void* object, uint32 flags) { + DepotMagazine* freeMagazine = NULL; + ReadLocker readLocker(depot->outer_lock); InterruptsLocker interruptsLocker; @@ -256,18 +265,31 @@ object_depot_store(object_depot* depot, void* object, uint32 flags) // we return the object directly to the slab. while (true) { - if (store->loaded && store->loaded->Push(object)) + if (store->loaded != NULL && store->loaded->Push(object)) return 1; - if ((store->previous && store->previous->IsEmpty()) - || exchange_with_empty(depot, store->previous)) { + if ((store->previous != NULL && store->previous->IsEmpty()) + || exchange_with_empty(depot, store->previous, freeMagazine)) { std::swap(store->loaded, store->previous); + + if (freeMagazine != NULL) { + // Free the magazine that didn't have space in the list + interruptsLocker.Unlock(); + readLocker.Unlock(); + + empty_magazine(depot, freeMagazine, flags); + + readLocker.Lock(); + interruptsLocker.Lock(); + + store = object_depot_cpu(depot); + } } else { // allocate a new empty magazine interruptsLocker.Unlock(); readLocker.Unlock(); - DepotMagazine* magazine = alloc_magazine(flags); + DepotMagazine* magazine = alloc_magazine(depot, flags); if (magazine == NULL) return 0; diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index bf9209998b..d4c9392684 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -413,6 +413,11 @@ object_cache_low_memory(void* dummy, uint32 resources, int32 level) if (cache->reclaimer) cache->reclaimer(cache->cookie, level); + if ((cache->flags & CACHE_NO_DEPOT) == 0) { + object_depot_make_empty(&cache->depot, 0); + // TODO: what flags? + } + MutexLocker cacheLocker(cache->lock); size_t minimumAllowed; @@ -540,16 +545,16 @@ create_object_cache(const char* name, size_t object_size, size_t alignment, void* cookie, object_cache_constructor constructor, object_cache_destructor destructor) { - return create_object_cache_etc(name, object_size, alignment, 0, 0, cookie, - constructor, destructor, NULL); + return create_object_cache_etc(name, object_size, alignment, 0, 0, 0, 0, + cookie, constructor, destructor, NULL); } object_cache* create_object_cache_etc(const char* name, size_t objectSize, size_t alignment, - size_t maximum, uint32 flags, void* cookie, - object_cache_constructor constructor, object_cache_destructor destructor, - object_cache_reclaimer reclaimer) + size_t maximum, size_t magazineCapacity, size_t maxMagazineCount, + uint32 flags, void* cookie, object_cache_constructor constructor, + object_cache_destructor destructor, object_cache_reclaimer reclaimer) { ObjectCache* cache; @@ -557,10 +562,12 @@ create_object_cache_etc(const char* name, size_t objectSize, size_t alignment, cache = NULL; } else if (objectSize <= 256) { cache = SmallObjectCache::Create(name, objectSize, alignment, maximum, - flags, cookie, constructor, destructor, reclaimer); + magazineCapacity, maxMagazineCount, flags, cookie, constructor, + destructor, reclaimer); } else { - cache = HashedObjectCache::Create(name, objectSize, alignment, - maximum, flags, cookie, constructor, destructor, reclaimer); + cache = HashedObjectCache::Create(name, objectSize, alignment, maximum, + magazineCapacity, maxMagazineCount, flags, cookie, constructor, + destructor, reclaimer); } if (cache != NULL) { diff --git a/src/system/kernel/slab/SmallObjectCache.cpp b/src/system/kernel/slab/SmallObjectCache.cpp index 62c0dbc661..8b0f3ee511 100644 --- a/src/system/kernel/slab/SmallObjectCache.cpp +++ b/src/system/kernel/slab/SmallObjectCache.cpp @@ -21,7 +21,8 @@ slab_in_pages(const void *pages, size_t slab_size) /*static*/ SmallObjectCache* SmallObjectCache::Create(const char* name, size_t object_size, - size_t alignment, size_t maximum, uint32 flags, void* cookie, + size_t alignment, size_t maximum, size_t magazineCapacity, + size_t maxMagazineCount, uint32 flags, void* cookie, object_cache_constructor constructor, object_cache_destructor destructor, object_cache_reclaimer reclaimer) { @@ -31,8 +32,9 @@ SmallObjectCache::Create(const char* name, size_t object_size, SmallObjectCache* cache = new(buffer) SmallObjectCache(); - if (cache->Init(name, object_size, alignment, maximum, flags, cookie, - constructor, destructor, reclaimer) != B_OK) { + if (cache->Init(name, object_size, alignment, maximum, magazineCapacity, + maxMagazineCount, flags, cookie, constructor, destructor, + reclaimer) != B_OK) { cache->Delete(); return NULL; } diff --git a/src/system/kernel/slab/SmallObjectCache.h b/src/system/kernel/slab/SmallObjectCache.h index e51182b082..2d19a82d62 100644 --- a/src/system/kernel/slab/SmallObjectCache.h +++ b/src/system/kernel/slab/SmallObjectCache.h @@ -14,6 +14,8 @@ struct SmallObjectCache : ObjectCache { static SmallObjectCache* Create(const char* name, size_t object_size, size_t alignment, size_t maximum, + size_t magazineCapacity, + size_t maxMagazineCount, uint32 flags, void* cookie, object_cache_constructor constructor, object_cache_destructor destructor, diff --git a/src/system/kernel/slab/allocator.cpp b/src/system/kernel/slab/allocator.cpp index 0f7954a2a3..2eedab0d55 100644 --- a/src/system/kernel/slab/allocator.cpp +++ b/src/system/kernel/slab/allocator.cpp @@ -180,8 +180,8 @@ block_allocator_init_boot() if (size > 2048) flags |= CACHE_NO_DEPOT; - sBlockCaches[index] = create_object_cache_etc(name, size, 0, 0, flags, - NULL, NULL, NULL, NULL); + sBlockCaches[index] = create_object_cache_etc(name, size, 0, 0, 0, 0, + flags, NULL, NULL, NULL, NULL); if (sBlockCaches[index] == NULL) panic("allocator: failed to init block cache"); } diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index e540d850e0..85b939add1 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -1,6 +1,6 @@ /* * Copyright 2009-2010, Ingo Weinhold, ingo_weinhold@gmx.de. - * Copyright 2002-2009, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. * * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. @@ -3395,8 +3395,8 @@ vm_init(kernel_args* args) // create the object cache for the page mappings gPageMappingsObjectCache = create_object_cache_etc("page mappings", - sizeof(vm_page_mapping), 0, 0, CACHE_LARGE_SLAB, NULL, NULL, NULL, - NULL); + sizeof(vm_page_mapping), 0, 0, 64, 128, CACHE_LARGE_SLAB, NULL, NULL, + NULL, NULL); if (gPageMappingsObjectCache == NULL) panic("failed to create page mappings object cache");