diff --git a/headers/private/kernel/slab/Depot.h b/headers/private/kernel/slab/ObjectDepot.h similarity index 89% rename from headers/private/kernel/slab/Depot.h rename to headers/private/kernel/slab/ObjectDepot.h index 02917e28de..be3f7762e6 100644 --- a/headers/private/kernel/slab/Depot.h +++ b/headers/private/kernel/slab/ObjectDepot.h @@ -2,8 +2,8 @@ * Copyright 2007, Hugo Santos. All Rights Reserved. * Distributed under the terms of the MIT License. */ -#ifndef _SLAB_DEPOT_H_ -#define _SLAB_DEPOT_H_ +#ifndef _SLAB_OBJECT_DEPOT_H_ +#define _SLAB_OBJECT_DEPOT_H_ #include @@ -37,4 +37,4 @@ void object_depot_make_empty(object_depot *depot); } #endif -#endif /* _SLAB_DEPOT_H_ */ +#endif /* _SLAB_OBJECT_DEPOT_H_ */ diff --git a/src/system/kernel/slab/Jamfile b/src/system/kernel/slab/Jamfile index 23b9cfaa83..0eec9c1035 100644 --- a/src/system/kernel/slab/Jamfile +++ b/src/system/kernel/slab/Jamfile @@ -1,10 +1,10 @@ SubDir HAIKU_TOP src system kernel slab ; -UsePrivateHeaders [ FDirName kernel slab ] ; KernelMergeObject kernel_slab.o : allocator.cpp + ObjectDepot.cpp Slab.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) - ; +; diff --git a/src/system/kernel/slab/ObjectDepot.cpp b/src/system/kernel/slab/ObjectDepot.cpp new file mode 100644 index 0000000000..9b5e1a79a0 --- /dev/null +++ b/src/system/kernel/slab/ObjectDepot.cpp @@ -0,0 +1,279 @@ +/* + * Copyright 2008, Axel Dörfler. All Rights Reserved. + * Copyright 2007, Hugo Santos. All Rights Reserved. + * + * Distributed under the terms of the MIT License. + */ + + +#include + +#include + +#include + +#include "slab_private.h" + + +static const int kMagazineCapacity = 32; + // TODO: Should be dynamically tuned per cache. + + +struct depot_magazine { + struct depot_magazine *next; + uint16 current_round, round_count; + void *rounds[0]; +}; + + +struct depot_cpu_store { + recursive_lock lock; + struct depot_magazine *loaded, *previous; +}; + + +static inline bool +is_magazine_empty(depot_magazine *magazine) +{ + return magazine->current_round == 0; +} + + +static inline bool +is_magazine_full(depot_magazine *magazine) +{ + return magazine->current_round == magazine->round_count; +} + + +static inline void * +pop_magazine(depot_magazine *magazine) +{ + return magazine->rounds[--magazine->current_round]; +} + + +static inline bool +push_magazine(depot_magazine *magazine, void *object) +{ + if (is_magazine_full(magazine)) + return false; + magazine->rounds[magazine->current_round++] = object; + return true; +} + + +static depot_magazine * +alloc_magazine() +{ + depot_magazine *magazine = (depot_magazine *)internal_alloc( + sizeof(depot_magazine) + kMagazineCapacity * sizeof(void *), 0); + if (magazine) { + magazine->next = NULL; + magazine->current_round = 0; + magazine->round_count = kMagazineCapacity; + } + + return magazine; +} + + +static void +free_magazine(depot_magazine *magazine) +{ + internal_free(magazine); +} + + +static void +empty_magazine(object_depot *depot, depot_magazine *magazine) +{ + for (uint16 i = 0; i < magazine->current_round; i++) + depot->return_object(depot, magazine->rounds[i]); + free_magazine(magazine); +} + + +static bool +exchange_with_full(object_depot *depot, depot_magazine* &magazine) +{ + RecursiveLocker _(depot->lock); + + if (depot->full == NULL) + return false; + + depot->full_count--; + depot->empty_count++; + + _push(depot->empty, magazine); + magazine = _pop(depot->full); + return true; +} + + +static bool +exchange_with_empty(object_depot *depot, depot_magazine* &magazine) +{ + RecursiveLocker _(depot->lock); + + if (depot->empty == NULL) { + depot->empty = alloc_magazine(); + if (depot->empty == NULL) + return false; + } else { + depot->empty_count--; + } + + if (magazine) { + _push(depot->full, magazine); + depot->full_count++; + } + + magazine = _pop(depot->empty); + return true; +} + + +static inline depot_cpu_store * +object_depot_cpu(object_depot *depot) +{ + return &depot->stores[smp_get_current_cpu()]; +} + + +// #pragma mark - + + +status_t +object_depot_init(object_depot *depot, uint32 flags, + void (*return_object)(object_depot *depot, void *object)) +{ + depot->full = NULL; + depot->empty = NULL; + depot->full_count = depot->empty_count = 0; + + recursive_lock_init(&depot->lock, "depot"); + + depot->stores = (depot_cpu_store *)internal_alloc(sizeof(depot_cpu_store) + * smp_get_num_cpus(), flags); + if (depot->stores == NULL) { + recursive_lock_destroy(&depot->lock); + return B_NO_MEMORY; + } + + for (int i = 0; i < smp_get_num_cpus(); i++) { + recursive_lock_init(&depot->stores[i].lock, "cpu store"); + depot->stores[i].loaded = depot->stores[i].previous = NULL; + } + + depot->return_object = return_object; + + return B_OK; +} + + +void +object_depot_destroy(object_depot *depot) +{ + object_depot_make_empty(depot); + + for (int i = 0; i < smp_get_num_cpus(); i++) { + recursive_lock_destroy(&depot->stores[i].lock); + } + + internal_free(depot->stores); + + recursive_lock_destroy(&depot->lock); +} + + +static void * +object_depot_obtain_from_store(object_depot *depot, depot_cpu_store *store) +{ + RecursiveLocker _(store->lock); + + // To better understand both the Alloc() and Free() logic refer to + // Bonwick's ``Magazines and Vmem'' [in 2001 USENIX proceedings] + + // In a nutshell, we try to get an object from the loaded magazine + // if it's not empty, or from the previous magazine if it's full + // and finally from the Slab if the magazine depot has no full magazines. + + if (store->loaded == NULL) + return NULL; + + while (true) { + if (!is_magazine_empty(store->loaded)) + return pop_magazine(store->loaded); + + if (store->previous && (is_magazine_full(store->previous) + || exchange_with_full(depot, store->previous))) + std::swap(store->previous, store->loaded); + else + return NULL; + } +} + + +static int +object_depot_return_to_store(object_depot *depot, depot_cpu_store *store, + void *object) +{ + RecursiveLocker _(store->lock); + + // We try to add the object to the loaded magazine if we have one + // and it's not full, or to the previous one if it is empty. If + // the magazine depot doesn't provide us with a new empty magazine + // we return the object directly to the slab. + + while (true) { + if (store->loaded && push_magazine(store->loaded, object)) + return 1; + + if ((store->previous && is_magazine_empty(store->previous)) + || exchange_with_empty(depot, store->previous)) + std::swap(store->loaded, store->previous); + else + return 0; + } +} + + +void * +object_depot_obtain(object_depot *depot) +{ + return object_depot_obtain_from_store(depot, object_depot_cpu(depot)); +} + + +int +object_depot_store(object_depot *depot, void *object) +{ + return object_depot_return_to_store(depot, object_depot_cpu(depot), + object); +} + + +void +object_depot_make_empty(object_depot *depot) +{ + for (int i = 0; i < smp_get_num_cpus(); i++) { + depot_cpu_store *store = &depot->stores[i]; + + RecursiveLocker _(store->lock); + + if (depot->stores[i].loaded) + empty_magazine(depot, depot->stores[i].loaded); + if (depot->stores[i].previous) + empty_magazine(depot, depot->stores[i].previous); + depot->stores[i].loaded = depot->stores[i].previous = NULL; + } + + RecursiveLocker _(depot->lock); + + while (depot->full) + empty_magazine(depot, _pop(depot->full)); + + while (depot->empty) + empty_magazine(depot, _pop(depot->empty)); +} diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp index ae82c6d252..f4b274a503 100644 --- a/src/system/kernel/slab/Slab.cpp +++ b/src/system/kernel/slab/Slab.cpp @@ -6,8 +6,7 @@ */ -#include -#include "slab_private.h" +#include #include #include @@ -18,9 +17,9 @@ #include #include -#include #include #include +#include #include #include #include @@ -30,8 +29,7 @@ #include #include - -// TODO kMagazineCapacity should be dynamically tuned per cache. +#include "slab_private.h" //#define TRACE_SLAB @@ -49,7 +47,6 @@ #define CACHE_ALIGN_ON_SIZE (30 << 1) -static const int kMagazineCapacity = 32; static const size_t kCacheColorPeriod = 8; struct object_link { @@ -166,18 +163,6 @@ struct HashedObjectCache : object_cache { }; -struct depot_magazine { - struct depot_magazine *next; - uint16 current_round, round_count; - void *rounds[0]; -}; - - -struct depot_cpu_store { - recursive_lock lock; - struct depot_magazine *loaded, *previous; -}; - struct ResizeRequest : DoublyLinkedListLinkImpl { ResizeRequest(object_cache* cache) : @@ -204,8 +189,6 @@ static kernel_args *sKernelArgs; static status_t object_cache_reserve_internal(object_cache *cache, size_t object_count, uint32 flags, bool unlockWhileAllocating); -static depot_magazine *alloc_magazine(); -static void free_magazine(depot_magazine *magazine); static mutex sResizeRequestsLock = MUTEX_INITIALIZER("object cache resize requests"); @@ -361,23 +344,6 @@ class Reserve : public ObjectCacheTraceEntry { // #pragma mark - -template static Type * -_pop(Type* &head) -{ - Type *oldHead = head; - head = head->next; - return oldHead; -} - - -template static inline void -_push(Type* &head, Type *object) -{ - object->next = head; - head = object; -} - - static inline void * link_to_object(object_link *link, size_t objectSize) { @@ -393,13 +359,6 @@ object_to_link(void *object, size_t objectSize) } -static inline depot_cpu_store * -object_depot_cpu(object_depot *depot) -{ - return &depot->stores[smp_get_current_cpu()]; -} - - static inline int __fls0(size_t value) { @@ -413,7 +372,7 @@ __fls0(size_t value) } -static void * +void * internal_alloc(size_t size, uint32 flags) { if (flags & CACHE_DURING_BOOT) { @@ -432,7 +391,7 @@ internal_alloc(size_t size, uint32 flags) } -static void +void internal_free(void *_buffer) { uint8 *buffer = (uint8 *)_buffer; @@ -1356,243 +1315,6 @@ HashedObjectCache::UnprepareObject(slab *source, void *object) } -static inline bool -is_magazine_empty(depot_magazine *magazine) -{ - return magazine->current_round == 0; -} - - -static inline bool -is_magazine_full(depot_magazine *magazine) -{ - return magazine->current_round == magazine->round_count; -} - - -static inline void * -pop_magazine(depot_magazine *magazine) -{ - return magazine->rounds[--magazine->current_round]; -} - - -static inline bool -push_magazine(depot_magazine *magazine, void *object) -{ - if (is_magazine_full(magazine)) - return false; - magazine->rounds[magazine->current_round++] = object; - return true; -} - - -static bool -exchange_with_full(object_depot *depot, depot_magazine* &magazine) -{ - RecursiveLocker _(depot->lock); - - if (depot->full == NULL) - return false; - - depot->full_count--; - depot->empty_count++; - - _push(depot->empty, magazine); - magazine = _pop(depot->full); - return true; -} - - -static bool -exchange_with_empty(object_depot *depot, depot_magazine* &magazine) -{ - RecursiveLocker _(depot->lock); - - if (depot->empty == NULL) { - depot->empty = alloc_magazine(); - if (depot->empty == NULL) - return false; - } else { - depot->empty_count--; - } - - if (magazine) { - _push(depot->full, magazine); - depot->full_count++; - } - - magazine = _pop(depot->empty); - return true; -} - - -static depot_magazine * -alloc_magazine() -{ - depot_magazine *magazine = (depot_magazine *)internal_alloc( - sizeof(depot_magazine) + kMagazineCapacity * sizeof(void *), 0); - if (magazine) { - magazine->next = NULL; - magazine->current_round = 0; - magazine->round_count = kMagazineCapacity; - } - - return magazine; -} - - -static void -free_magazine(depot_magazine *magazine) -{ - internal_free(magazine); -} - - -static void -empty_magazine(object_depot *depot, depot_magazine *magazine) -{ - for (uint16 i = 0; i < magazine->current_round; i++) - depot->return_object(depot, magazine->rounds[i]); - free_magazine(magazine); -} - - -status_t -object_depot_init(object_depot *depot, uint32 flags, - void (*return_object)(object_depot *depot, void *object)) -{ - depot->full = NULL; - depot->empty = NULL; - depot->full_count = depot->empty_count = 0; - - recursive_lock_init(&depot->lock, "depot"); - - depot->stores = (depot_cpu_store *)internal_alloc(sizeof(depot_cpu_store) - * smp_get_num_cpus(), flags); - if (depot->stores == NULL) { - recursive_lock_destroy(&depot->lock); - return B_NO_MEMORY; - } - - for (int i = 0; i < smp_get_num_cpus(); i++) { - recursive_lock_init(&depot->stores[i].lock, "cpu store"); - depot->stores[i].loaded = depot->stores[i].previous = NULL; - } - - depot->return_object = return_object; - - return B_OK; -} - - -void -object_depot_destroy(object_depot *depot) -{ - object_depot_make_empty(depot); - - for (int i = 0; i < smp_get_num_cpus(); i++) { - recursive_lock_destroy(&depot->stores[i].lock); - } - - internal_free(depot->stores); - - recursive_lock_destroy(&depot->lock); -} - - -static void * -object_depot_obtain_from_store(object_depot *depot, depot_cpu_store *store) -{ - RecursiveLocker _(store->lock); - - // To better understand both the Alloc() and Free() logic refer to - // Bonwick's ``Magazines and Vmem'' [in 2001 USENIX proceedings] - - // In a nutshell, we try to get an object from the loaded magazine - // if it's not empty, or from the previous magazine if it's full - // and finally from the Slab if the magazine depot has no full magazines. - - if (store->loaded == NULL) - return NULL; - - while (true) { - if (!is_magazine_empty(store->loaded)) - return pop_magazine(store->loaded); - - if (store->previous && (is_magazine_full(store->previous) - || exchange_with_full(depot, store->previous))) - std::swap(store->previous, store->loaded); - else - return NULL; - } -} - - -static int -object_depot_return_to_store(object_depot *depot, depot_cpu_store *store, - void *object) -{ - RecursiveLocker _(store->lock); - - // We try to add the object to the loaded magazine if we have one - // and it's not full, or to the previous one if it is empty. If - // the magazine depot doesn't provide us with a new empty magazine - // we return the object directly to the slab. - - while (true) { - if (store->loaded && push_magazine(store->loaded, object)) - return 1; - - if ((store->previous && is_magazine_empty(store->previous)) - || exchange_with_empty(depot, store->previous)) - std::swap(store->loaded, store->previous); - else - return 0; - } -} - - -void * -object_depot_obtain(object_depot *depot) -{ - return object_depot_obtain_from_store(depot, object_depot_cpu(depot)); -} - - -int -object_depot_store(object_depot *depot, void *object) -{ - return object_depot_return_to_store(depot, object_depot_cpu(depot), - object); -} - - -void -object_depot_make_empty(object_depot *depot) -{ - for (int i = 0; i < smp_get_num_cpus(); i++) { - depot_cpu_store *store = &depot->stores[i]; - - RecursiveLocker _(store->lock); - - if (depot->stores[i].loaded) - empty_magazine(depot, depot->stores[i].loaded); - if (depot->stores[i].previous) - empty_magazine(depot, depot->stores[i].previous); - depot->stores[i].loaded = depot->stores[i].previous = NULL; - } - - RecursiveLocker _(depot->lock); - - while (depot->full) - empty_magazine(depot, _pop(depot->full)); - - while (depot->empty) - empty_magazine(depot, _pop(depot->empty)); -} - - static int dump_slabs(int argc, char *argv[]) { diff --git a/src/system/kernel/slab/allocator.cpp b/src/system/kernel/slab/allocator.cpp index d557e20925..5e19716e85 100644 --- a/src/system/kernel/slab/allocator.cpp +++ b/src/system/kernel/slab/allocator.cpp @@ -6,15 +6,16 @@ * Hugo Santos, hugosantos@gmail.com */ -#include - -#include "slab_private.h" +#include #include // for ROUNDUP #include #include +#include "slab_private.h" + + #define DEBUG_ALLOCATOR //#define TEST_ALL_CACHES_DURING_BOOT diff --git a/src/system/kernel/slab/slab_private.h b/src/system/kernel/slab/slab_private.h index 883712f7c5..73c04c9588 100644 --- a/src/system/kernel/slab/slab_private.h +++ b/src/system/kernel/slab/slab_private.h @@ -5,15 +5,40 @@ * Authors: * Hugo Santos, hugosantos@gmail.com */ -#ifndef _SLAB_PRIVATE_H_ -#define _SLAB_PRIVATE_H_ +#ifndef SLAB_PRIVATE_H +#define SLAB_PRIVATE_H + #include -void *block_alloc(size_t size); -void block_free(void *block); -void block_allocator_init_boot(); -void block_allocator_init_rest(); -#endif +void* internal_alloc(size_t size, uint32 flags); +void internal_free(void *_buffer); + +void* block_alloc(size_t size); +void block_free(void *block); +void block_allocator_init_boot(); +void block_allocator_init_rest(); + + +template +static inline Type* +_pop(Type*& head) +{ + Type* oldHead = head; + head = head->next; + return oldHead; +} + + +template +static inline void +_push(Type*& head, Type* object) +{ + object->next = head; + head = object; +} + + +#endif // SLAB_PRIVATE_H