mirrors/mimalloc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more eager handling of non-local frees

Browse Source
This commit is contained in:
daan 2019-07-11 15:21:57 -07:00
parent 72a39c0bb1
commit 1fdb4b288f
5 changed files with 44 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/mimalloc-internal.h
View File

@ -66,7 +66,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); //
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
void _mi_heap_delayed_free(mi_heap_t* heap);
void _mi_page_use_delayed_free(mi_page_t* page, bool enable);
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay);
size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);
void _mi_deferred_free(mi_heap_t* heap, bool force);

7
include/mimalloc-types.h
View File

@ -114,8 +114,9 @@ typedef struct mi_block_s {
typedef enum mi_delayed_e {
MI_NO_DELAYED_FREE = 0,
MI_USE_DELAYED_FREE,
MI_DELAYED_FREEING
MI_USE_DELAYED_FREE = 1,
MI_DELAYED_FREEING = 2,
MI_NEVER_DELAYED_FREE = 3
} mi_delayed_t;
@ -132,7 +133,7 @@ typedef union mi_page_flags_u {
typedef union mi_thread_free_u {
volatile uintptr_t value;
struct {
mi_delayed_t delayed:2;
uintptr_t delayed:2;
#if MI_INTPTR_SIZE==8
uintptr_t head:62; // head free block in the list (right-shifted by 2)
#elif MI_INTPTR_SIZE==4

7
src/alloc.c
View File

@ -112,7 +112,9 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
do {
tfreex.value = tfree.value = page->thread_free.value;
use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE);
use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE ||
(tfree.delayed == MI_NO_DELAYED_FREE && page->used == page->thread_freed+1)
);
if (mi_unlikely(use_delayed)) {
// unlikely: this only happens on the first concurrent free in a page that is in the full list
tfreex.delayed = MI_DELAYED_FREEING;
@ -144,7 +146,8 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
// and reset the MI_DELAYED_FREEING flag
do {
tfreex.value = tfree.value = page->thread_free.value;
tfreex.delayed = MI_NO_DELAYED_FREE;
mi_assert_internal(tfree.delayed == MI_NEVER_DELAYED_FREE || tfree.delayed == MI_DELAYED_FREEING);
if (tfree.delayed != MI_NEVER_DELAYED_FREE) tfreex.delayed = MI_NO_DELAYED_FREE;
} while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value));
}
}

19
src/heap.c
View File

@ -97,6 +97,14 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t
return true; // don't break
}
static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
UNUSED(arg1);
UNUSED(arg2);
UNUSED(heap);
UNUSED(pq);
_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE);
return true; // don't break
}
static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
{
@ -119,11 +127,12 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
#endif
}
// if abandoning, mark all full pages to no longer add to delayed_free
// if abandoning, mark all pages to no longer add to delayed_free
if (collect == ABANDON) {
for (mi_page_t* page = heap->pages[MI_BIN_FULL].first; page != NULL; page = page->next) {
_mi_page_use_delayed_free(page, false); // set thread_free.delayed to MI_NO_DELAYED_FREE
}
//for (mi_page_t* page = heap->pages[MI_BIN_FULL].first; page != NULL; page = page->next) {
// _mi_page_use_delayed_free(page, false); // set thread_free.delayed to MI_NO_DELAYED_FREE
//}
mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL);
}
// free thread delayed blocks.
@ -228,7 +237,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
UNUSED(pq);
// ensure no more thread_delayed_free will be added
_mi_page_use_delayed_free(page, false);
_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE);
// stats
if (page->block_size > MI_LARGE_SIZE_MAX) {

29
src/page.c
View File

@ -109,17 +109,19 @@ bool _mi_page_is_valid(mi_page_t* page) {
#endif
void _mi_page_use_delayed_free(mi_page_t* page, bool enable) {
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay ) {
mi_thread_free_t tfree;
mi_thread_free_t tfreex;
do {
tfreex.value = tfree.value = page->thread_free.value;
tfreex.delayed = (enable ? MI_USE_DELAYED_FREE : MI_NO_DELAYED_FREE);
if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) {
if (mi_unlikely(tfree.delayed < MI_DELAYED_FREEING)) {
tfreex.delayed = delay;
}
else if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) {
mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
continue; // and try again
}
}
}
while(tfreex.delayed != tfree.delayed && // avoid atomic operation if already equal
!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value));
@ -272,7 +274,7 @@ void _mi_page_unfull(mi_page_t* page) {
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(page->flags.in_full);
_mi_page_use_delayed_free(page, false);
_mi_page_use_delayed_free(page, MI_NO_DELAYED_FREE);
if (!page->flags.in_full) return;
mi_heap_t* heap = page->heap;
@ -288,7 +290,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
mi_assert_internal(!mi_page_immediate_available(page));
mi_assert_internal(!page->flags.in_full);
_mi_page_use_delayed_free(page, true);
_mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE);
if (page->flags.in_full) return;
mi_page_queue_enqueue_from(&page->heap->pages[MI_BIN_FULL], pq, page);
@ -305,8 +307,8 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(page->heap != NULL);
mi_assert_internal(page->thread_free.delayed == MI_NO_DELAYED_FREE);
_mi_page_use_delayed_free(page,MI_NEVER_DELAYED_FREE);
#if MI_DEBUG>1
// check there are no references left..
for (mi_block_t* block = (mi_block_t*)page->heap->thread_delayed_free; block != NULL; block = mi_block_nextx(page->heap->cookie,block)) {
@ -330,7 +332,14 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(mi_page_all_free(page));
mi_assert_internal(page->thread_free.delayed != MI_DELAYED_FREEING);
#if MI_DEBUG>1
// check if we can safely free
mi_thread_free_t free;
free.value = page->thread_free.value;
free.delayed = MI_NEVER_DELAYED_FREE;
free.value = mi_atomic_exchange(&page->thread_free.value, free.value);
mi_assert_internal(free.delayed != MI_DELAYED_FREEING);
#endif
page->flags.has_aligned = false;
@ -717,10 +726,12 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
mi_assert_internal(page->block_size >= size);
// and try again, this time succeeding! (i.e. this should never recurse)
void* p = _mi_page_malloc(heap, page, size);
return _mi_page_malloc(heap, page, size);
/*
if (page->used == page->reserved) {
// needed for huge pages to free reliably from other threads.
mi_page_to_full(page,mi_page_queue_of(page));
}
return p;
*/
}