From 23812cc0ac0e37fb2c123f1d391aecdfc372fbfc Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 24 Aug 2019 15:45:14 -0700 Subject: [PATCH] do not keep a queue of huge pages and free them directly --- src/alloc.c | 15 +++++++++++++++ src/page-queue.c | 1 + src/page.c | 30 ++++++++++++++++++------------ src/segment.c | 5 +++-- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index b7881ea5..76e093e7 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -115,6 +115,21 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc mi_thread_free_t tfreex; bool use_delayed; + mi_segment_t* segment = _mi_page_segment(page); + if (segment->page_kind==MI_PAGE_HUGE) { + // huge page segments are always abandoned and can be freed immediately + mi_assert_internal(segment->thread_id==0); + mi_assert_internal(segment->abandoned_next==NULL); + // claim it and free + mi_block_set_next(page, block, page->free); + page->free = block; + page->used--; + mi_heap_t* heap = mi_get_default_heap(); + segment->thread_id = heap->thread_id; + _mi_segment_page_free(page,true,&heap->tld->segments); + return; + } + do { tfree = page->thread_free; use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE || diff --git a/src/page-queue.c b/src/page-queue.c index 859b1d57..d613095f 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -268,6 +268,7 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(page->heap == NULL); mi_assert_internal(!mi_page_queue_contains(queue, page)); + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); diff --git a/src/page.c b/src/page.c index a7b4a760..f7e0ce2c 100644 --- a/src/page.c +++ b/src/page.c @@ -98,11 +98,13 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif if (page->heap!=NULL) { mi_segment_t* segment = _mi_page_segment(page); - mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id); - mi_page_queue_t* pq = mi_page_queue_of(page); - mi_assert_internal(mi_page_queue_contains(pq, page)); - mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); - mi_assert_internal(mi_heap_contains_queue(page->heap,pq)); + mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id || segment->thread_id==0); + if (segment->page_kind != MI_PAGE_HUGE) { + mi_page_queue_t* pq = mi_page_queue_of(page); + mi_assert_internal(mi_page_queue_contains(pq, page)); + mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); + mi_assert_internal(mi_heap_contains_queue(page->heap,pq)); + } } return true; } @@ -204,6 +206,7 @@ void _mi_page_free_collect(mi_page_t* page) { void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); mi_assert_internal(page->heap == NULL); + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); _mi_page_free_collect(page); mi_page_queue_t* pq = mi_page_queue(heap, page->block_size); mi_page_queue_push(heap, pq, page); @@ -212,12 +215,13 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { // allocate a fresh page from a segment static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size_t block_size) { - mi_assert_internal(mi_heap_contains_queue(heap, pq)); + mi_assert_internal(pq==NULL||mi_heap_contains_queue(heap, pq)); mi_page_t* page = _mi_segment_page_alloc(block_size, &heap->tld->segments, &heap->tld->os); if (page == NULL) return NULL; + mi_assert_internal(pq==NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE); mi_page_init(heap, page, block_size, &heap->tld->stats); _mi_stat_increase( &heap->tld->stats.pages, 1); - mi_page_queue_push(heap, pq, page); + if (pq!=NULL) mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL mi_assert_expensive(_mi_page_is_valid(page)); return page; } @@ -699,13 +703,15 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn) mi_attr_noexcept { // A huge page is allocated directly without being in a queue static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { size_t block_size = _mi_wsize_from_size(size) * sizeof(uintptr_t); - mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); - mi_page_queue_t* pq = mi_page_queue(heap,block_size); - mi_assert_internal(mi_page_queue_is_huge(pq)); - mi_page_t* page = mi_page_fresh_alloc(heap,pq,block_size); + mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); + mi_page_t* page = mi_page_fresh_alloc(heap,NULL,block_size); if (page != NULL) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); + mi_assert_internal(_mi_page_segment(page)->page_kind==MI_PAGE_HUGE); + mi_assert_internal(_mi_page_segment(page)->used==1); + mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue + page->heap = NULL; if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_increase(&heap->tld->stats.giant, block_size); _mi_stat_counter_increase(&heap->tld->stats.giant_count, 1); @@ -714,7 +720,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { _mi_stat_increase(&heap->tld->stats.huge, block_size); _mi_stat_counter_increase(&heap->tld->stats.huge_count, 1); } - } + } return page; } diff --git a/src/segment.c b/src/segment.c index b1a5221c..3be703cf 100644 --- a/src/segment.c +++ b/src/segment.c @@ -134,7 +134,7 @@ static bool mi_segment_is_valid(mi_segment_t* segment) { if (!segment->pages[i].segment_in_use) nfree++; } mi_assert_internal(nfree + segment->used == segment->capacity); - mi_assert_internal(segment->thread_id == _mi_thread_id()); // or 0 + mi_assert_internal(segment->thread_id == _mi_thread_id() || (segment->thread_id==0)); // or 0 mi_assert_internal(segment->page_kind == MI_PAGE_HUGE || (mi_segment_pagesize(segment) * segment->capacity == segment->segment_size)); return true; @@ -700,6 +700,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld if (segment == NULL) return NULL; mi_assert_internal(segment->segment_size - segment->segment_info_size >= size); segment->used = 1; + segment->thread_id = 0; // huge pages are immediately abandoned mi_page_t* page = &segment->pages[0]; page->segment_in_use = true; return page; @@ -721,7 +722,7 @@ mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_ else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { page = mi_segment_medium_page_alloc(tld, os_tld); } - else if (block_size < MI_LARGE_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t))) { + else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) { page = mi_segment_large_page_alloc(tld, os_tld); } else {