diff --git a/include/mimalloc.h b/include/mimalloc.h index a5b3cc9d..df85a2c0 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -369,6 +369,7 @@ typedef enum mi_option_e { mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) mi_option_debug_guarded_min, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) mi_option_debug_guarded_max, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) + mi_option_target_segments_per_thread, // experimental (=0) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index aff6a1bd..b4e74789 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -175,6 +175,8 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; / void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... +void _mi_page_force_abandon(mi_page_t* page); + void _mi_heap_delayed_free_all(mi_heap_t* heap); bool _mi_heap_delayed_free_partial(mi_heap_t* heap); void _mi_heap_collect_retired(mi_heap_t* heap, bool force); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 69f737b3..044d6eae 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -200,7 +200,7 @@ typedef int32_t mi_ssize_t; #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 8KiB on 64-bit #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit #define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE) -#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit +#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 16MiB on 64-bit #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) // Maximum number of size classes. (spaced exponentially in 12.5% increments) diff --git a/src/arena-abandon.c b/src/arena-abandon.c index eaa8c7c9..84b9f72c 100644 --- a/src/arena-abandon.c +++ b/src/arena-abandon.c @@ -192,7 +192,7 @@ void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool vi else { // otherwise visit all starting at a random location if (abandoned_count > abandoned_list_count && max_arena > 0) { - current->start = (heap == NULL || max_arena == 0 ? 0 : (mi_arena_id_t)(_mi_heap_random_next(heap) % max_arena)); + current->start = 0; // (heap == NULL || max_arena == 0 ? 0 : (mi_arena_id_t)(_mi_heap_random_next(heap) % max_arena)); current->end = current->start + max_arena; } else { diff --git a/src/options.c b/src/options.c index 1cfb2f17..c97b9abe 100644 --- a/src/options.c +++ b/src/options.c @@ -100,6 +100,7 @@ static mi_option_desc_t options[_mi_option_last] = #endif { 0, UNINIT, MI_OPTION(debug_guarded_min) }, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects { 0, UNINIT, MI_OPTION(debug_guarded_max) }, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects + { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable. }; static void mi_option_init(mi_option_desc_t* desc); diff --git a/src/page.c b/src/page.c index 99ef3835..5671c7d4 100644 --- a/src/page.c +++ b/src/page.c @@ -405,6 +405,27 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { } +// force abandon a page; this is safe to call +void _mi_page_force_abandon(mi_page_t* page) { + mi_heap_t* heap = mi_page_heap(page); + // mark page as not using delayed free + _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); + + // ensure this page is no longer in the heap delayed free list + _mi_heap_delayed_free_all(heap); + if (page->block_size == 0) return; // it may have been freed now + + // and now unlink it from the page queue and abandon (or free) + mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page); + if (mi_page_all_free(page)) { + _mi_page_free(page, pq, false); + } + else { + _mi_page_abandon(page, pq); + } +} + + // Free a page with no more free blocks void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_assert_internal(page != NULL); diff --git a/src/segment.c b/src/segment.c index 1f1dc006..bb7483f1 100644 --- a/src/segment.c +++ b/src/segment.c @@ -693,6 +693,8 @@ static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_ // free previous slice -- remove it from free and merge mi_assert_internal(prev->slice_count > 0 && prev->slice_offset==0); slice_count += prev->slice_count; + slice->slice_count = 0; + slice->slice_offset = (uint32_t)((uint8_t*)slice - (uint8_t*)prev); // set the slice offset for `segment_force_abandon` (in case the previous free block is very large). if (!is_abandoned) { mi_segment_span_remove_from_queue(prev, tld); } slice = prev; } @@ -1329,7 +1331,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice result = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); break; } - else if (segment->abandoned_visits > 3 && is_suitable) { + else if (segment->abandoned_visits > 3 && is_suitable && !mi_option_is_enabled(mi_option_target_segments_per_thread)) { // always reclaim on 3rd visit to limit the abandoned queue length. mi_segment_reclaim(segment, heap, 0, NULL, tld); } @@ -1343,7 +1345,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice return result; } - +// collect abandoned segments void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) { mi_segment_t* segment; @@ -1367,6 +1369,80 @@ void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) _mi_arena_field_cursor_done(¤t); } +/* ----------------------------------------------------------- + Force abandon a segment that is in use by our thread +----------------------------------------------------------- */ + +// force abandon a segment +static void mi_segment_force_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) +{ + mi_assert_internal(!mi_segment_is_abandoned(segment)); + + // for all slices + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + if (mi_slice_is_used(slice)) { + // ensure used count is up to date and collect potential concurrent frees + mi_page_t* const page = mi_slice_to_page(slice); + _mi_page_free_collect(page, false); + { + // abandon the page if it is still in-use (this will free it if possible as well) + mi_assert_internal(segment->used > 0); + if (segment->used == segment->abandoned+1) { + // the last page.. abandon and return as the segment will be abandoned after this + // and we should no longer access it. + _mi_page_force_abandon(page); + return; + } + else { + // abandon and continue + _mi_page_force_abandon(page); + // it might be freed, reset the slice (note: relies on coalesce setting the slice_offset) + slice = mi_slice_first(slice); + } + } + } + slice = slice + slice->slice_count; + } + mi_assert(segment->used == segment->abandoned); + mi_assert(segment->used == 0); + if (segment->used == 0) { + // all free now + mi_segment_free(segment, false, tld); + } + else { + // perform delayed purges + mi_segment_try_purge(segment, false /* force? */, tld->stats); + } +} + + +// try abandon segments. +// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use. +static void mi_segments_try_abandon(mi_heap_t* heap, mi_segments_tld_t* tld) { + const size_t target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread,0,1024); + if (target == 0 || tld->count <= target) return; + + const size_t min_target = (target > 4 ? (target*3)/4 : target); // 75% + + // todo: we should maintain a list of segments per thread; for now, only consider segments from the heap full pages + for (int i = 0; i < 16 && tld->count >= min_target; i++) { + mi_page_t* page = heap->pages[MI_BIN_FULL].first; + while (page != NULL && mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX) { + page = page->next; + } + if (page==NULL) { + break; + } + mi_segment_t* segment = _mi_page_segment(page); + mi_segment_force_abandon(segment, tld); + mi_assert_internal(page != heap->pages[MI_BIN_FULL].first); // as it is just abandoned + } +} + /* ----------------------------------------------------------- Reclaim or allocate ----------------------------------------------------------- */ @@ -1375,6 +1451,9 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_ { mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX); + // try to abandon some segments to increase reuse between threads + mi_segments_try_abandon(heap,tld); + // 1. try to reclaim an abandoned segment bool reclaimed; mi_segment_t* segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld);