From 1fdb4b288fcbc754487374028c58b0035fab9fce Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 11 Jul 2019 15:21:57 -0700 Subject: [PATCH] more eager handling of non-local frees --- include/mimalloc-internal.h | 2 +- include/mimalloc-types.h | 7 ++++--- src/alloc.c | 7 +++++-- src/heap.c | 19 ++++++++++++++----- src/page.c | 29 ++++++++++++++++++++--------- 5 files changed, 44 insertions(+), 20 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 64a169f8..9fa5993b 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -66,7 +66,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... void _mi_heap_delayed_free(mi_heap_t* heap); -void _mi_page_use_delayed_free(mi_page_t* page, bool enable); +void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay); size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); void _mi_deferred_free(mi_heap_t* heap, bool force); diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 145ea9f0..067616fe 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -114,8 +114,9 @@ typedef struct mi_block_s { typedef enum mi_delayed_e { MI_NO_DELAYED_FREE = 0, - MI_USE_DELAYED_FREE, - MI_DELAYED_FREEING + MI_USE_DELAYED_FREE = 1, + MI_DELAYED_FREEING = 2, + MI_NEVER_DELAYED_FREE = 3 } mi_delayed_t; @@ -132,7 +133,7 @@ typedef union mi_page_flags_u { typedef union mi_thread_free_u { volatile uintptr_t value; struct { - mi_delayed_t delayed:2; + uintptr_t delayed:2; #if MI_INTPTR_SIZE==8 uintptr_t head:62; // head free block in the list (right-shifted by 2) #elif MI_INTPTR_SIZE==4 diff --git a/src/alloc.c b/src/alloc.c index 8ae29723..6a2a263f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -112,7 +112,9 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc do { tfreex.value = tfree.value = page->thread_free.value; - use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE); + use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE || + (tfree.delayed == MI_NO_DELAYED_FREE && page->used == page->thread_freed+1) + ); if (mi_unlikely(use_delayed)) { // unlikely: this only happens on the first concurrent free in a page that is in the full list tfreex.delayed = MI_DELAYED_FREEING; @@ -144,7 +146,8 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc // and reset the MI_DELAYED_FREEING flag do { tfreex.value = tfree.value = page->thread_free.value; - tfreex.delayed = MI_NO_DELAYED_FREE; + mi_assert_internal(tfree.delayed == MI_NEVER_DELAYED_FREE || tfree.delayed == MI_DELAYED_FREEING); + if (tfree.delayed != MI_NEVER_DELAYED_FREE) tfreex.delayed = MI_NO_DELAYED_FREE; } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value)); } } diff --git a/src/heap.c b/src/heap.c index b7f93b8a..dc21bd0a 100644 --- a/src/heap.c +++ b/src/heap.c @@ -97,6 +97,14 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t return true; // don't break } +static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { + UNUSED(arg1); + UNUSED(arg2); + UNUSED(heap); + UNUSED(pq); + _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE); + return true; // don't break +} static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) { @@ -119,11 +127,12 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) #endif } - // if abandoning, mark all full pages to no longer add to delayed_free + // if abandoning, mark all pages to no longer add to delayed_free if (collect == ABANDON) { - for (mi_page_t* page = heap->pages[MI_BIN_FULL].first; page != NULL; page = page->next) { - _mi_page_use_delayed_free(page, false); // set thread_free.delayed to MI_NO_DELAYED_FREE - } + //for (mi_page_t* page = heap->pages[MI_BIN_FULL].first; page != NULL; page = page->next) { + // _mi_page_use_delayed_free(page, false); // set thread_free.delayed to MI_NO_DELAYED_FREE + //} + mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); } // free thread delayed blocks. @@ -228,7 +237,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ UNUSED(pq); // ensure no more thread_delayed_free will be added - _mi_page_use_delayed_free(page, false); + _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE); // stats if (page->block_size > MI_LARGE_SIZE_MAX) { diff --git a/src/page.c b/src/page.c index 1d5f77ae..e888885b 100644 --- a/src/page.c +++ b/src/page.c @@ -109,17 +109,19 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif -void _mi_page_use_delayed_free(mi_page_t* page, bool enable) { +void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay ) { mi_thread_free_t tfree; mi_thread_free_t tfreex; do { tfreex.value = tfree.value = page->thread_free.value; - tfreex.delayed = (enable ? MI_USE_DELAYED_FREE : MI_NO_DELAYED_FREE); - if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) { + if (mi_unlikely(tfree.delayed < MI_DELAYED_FREEING)) { + tfreex.delayed = delay; + } + else if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) { mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. continue; // and try again - } + } } while(tfreex.delayed != tfree.delayed && // avoid atomic operation if already equal !mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value)); @@ -272,7 +274,7 @@ void _mi_page_unfull(mi_page_t* page) { mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_internal(page->flags.in_full); - _mi_page_use_delayed_free(page, false); + _mi_page_use_delayed_free(page, MI_NO_DELAYED_FREE); if (!page->flags.in_full) return; mi_heap_t* heap = page->heap; @@ -288,7 +290,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { mi_assert_internal(!mi_page_immediate_available(page)); mi_assert_internal(!page->flags.in_full); - _mi_page_use_delayed_free(page, true); + _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE); if (page->flags.in_full) return; mi_page_queue_enqueue_from(&page->heap->pages[MI_BIN_FULL], pq, page); @@ -305,8 +307,8 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(page->heap != NULL); - mi_assert_internal(page->thread_free.delayed == MI_NO_DELAYED_FREE); + _mi_page_use_delayed_free(page,MI_NEVER_DELAYED_FREE); #if MI_DEBUG>1 // check there are no references left.. for (mi_block_t* block = (mi_block_t*)page->heap->thread_delayed_free; block != NULL; block = mi_block_nextx(page->heap->cookie,block)) { @@ -330,7 +332,14 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(mi_page_all_free(page)); - mi_assert_internal(page->thread_free.delayed != MI_DELAYED_FREEING); + #if MI_DEBUG>1 + // check if we can safely free + mi_thread_free_t free; + free.value = page->thread_free.value; + free.delayed = MI_NEVER_DELAYED_FREE; + free.value = mi_atomic_exchange(&page->thread_free.value, free.value); + mi_assert_internal(free.delayed != MI_DELAYED_FREEING); + #endif page->flags.has_aligned = false; @@ -717,10 +726,12 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_assert_internal(page->block_size >= size); // and try again, this time succeeding! (i.e. this should never recurse) - void* p = _mi_page_malloc(heap, page, size); + return _mi_page_malloc(heap, page, size); + /* if (page->used == page->reserved) { // needed for huge pages to free reliably from other threads. mi_page_to_full(page,mi_page_queue_of(page)); } return p; + */ }