diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index e261dba2..cc487a21 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -307,7 +307,7 @@ static inline bool mi_page_all_used(mi_page_t* page) { static inline bool mi_page_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; - return (page->reserved - page->used + page->thread_freed < frac); + return (page->reserved - page->used + page->thread_freed <= frac); } static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 5c14ffd4..6c094091 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -324,12 +324,12 @@ typedef struct mi_stats_s { mi_stat_count_t pages_abandoned; mi_stat_count_t pages_extended; mi_stat_count_t mmap_calls; - mi_stat_count_t mmap_right_align; - mi_stat_count_t mmap_ensure_aligned; mi_stat_count_t commit_calls; mi_stat_count_t threads; mi_stat_count_t huge; mi_stat_count_t malloc; + mi_stat_count_t segments_cache; + mi_stat_counter_t page_no_retire; mi_stat_counter_t searches; #if MI_STAT>1 mi_stat_count_t normal[MI_BIN_HUGE+1]; diff --git a/src/init.c b/src/init.c index 8075ea35..3b060fa4 100644 --- a/src/init.c +++ b/src/init.c @@ -61,7 +61,8 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), \ + { 0, 0 }, \ { 0, 0 } \ MI_STAT_COUNT_END_NULL() diff --git a/src/page.c b/src/page.c index b2bbc2ad..aa8a8415 100644 --- a/src/page.c +++ b/src/page.c @@ -216,7 +216,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size mi_page_t* page = _mi_segment_page_alloc(block_size, &heap->tld->segments, &heap->tld->os); if (page == NULL) return NULL; mi_page_init(heap, page, block_size, &heap->tld->stats); - mi_heap_stat_increase( heap, pages, 1); + _mi_stat_increase( &heap->tld->stats.pages, 1); mi_page_queue_push(heap, pq, page); mi_assert_expensive(_mi_page_is_valid(page)); return page; @@ -352,7 +352,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // account for huge pages here if (page->block_size > MI_LARGE_SIZE_MAX) { - mi_heap_stat_decrease(page->heap, huge, page->block_size); + _mi_stat_decrease(&page->heap->tld->stats.huge, page->block_size); } // remove from the page list @@ -386,6 +386,7 @@ void _mi_page_retire(mi_page_t* page) { // if its neighbours are almost fully used. if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { + _mi_stat_counter_increase(&page->heap->tld->stats.page_no_retire,1); return; // dont't retire after all } } @@ -700,7 +701,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { if (page != NULL) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); - mi_heap_stat_increase( heap, huge, block_size); + _mi_stat_increase( &heap->tld->stats.huge, block_size); } return page; } diff --git a/src/segment.c b/src/segment.c index 8f254a26..a86c3bc0 100644 --- a/src/segment.c +++ b/src/segment.c @@ -248,17 +248,19 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t tld->cache = segment->next; segment->next = NULL; mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); + _mi_stat_decrease(&tld->stats->segments_cache, 1); return segment; } static bool mi_segment_cache_full(mi_segments_tld_t* tld) { - if (tld->cache_count < MI_SEGMENT_CACHE_MAX && - tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { // always allow 1 element cache + if (tld->cache_count < MI_SEGMENT_CACHE_MAX + && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION)) + ) { // always allow 1 element cache return false; } // take the opportunity to reduce the segment cache if it is too large (now) // TODO: this never happens as we check against peak usage, should we use current usage instead? - while (tld->cache_count > (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { + while (tld->cache_count > MI_SEGMENT_CACHE_MAX ) { //(1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { mi_segment_t* segment = mi_segment_cache_pop(0,tld); mi_assert_internal(segment != NULL); if (segment != NULL) mi_segment_os_free(segment, segment->segment_size, tld); @@ -269,7 +271,9 @@ static bool mi_segment_cache_full(mi_segments_tld_t* tld) { static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) { mi_assert_internal(!mi_segment_is_in_free_queue(segment, tld)); mi_assert_internal(segment->next == NULL); - if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) return false; + if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) { + return false; + } mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); if (mi_option_is_enabled(mi_option_cache_reset)) { _mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); @@ -277,6 +281,7 @@ static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) segment->next = tld->cache; tld->cache = segment; tld->cache_count++; + _mi_stat_increase(&tld->stats->segments_cache,1); return true; } diff --git a/src/stats.c b/src/stats.c index 2b15bf9e..8725e48c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -99,14 +99,14 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); mi_stat_add(&stats->mmap_calls, &src->mmap_calls, 1); - mi_stat_add(&stats->mmap_ensure_aligned, &src->mmap_ensure_aligned, 1); - mi_stat_add(&stats->mmap_right_align, &src->mmap_right_align, 1); mi_stat_add(&stats->commit_calls, &src->commit_calls, 1); mi_stat_add(&stats->threads, &src->threads, 1); mi_stat_add(&stats->pages_extended, &src->pages_extended, 1); mi_stat_add(&stats->malloc, &src->malloc, 1); + mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); mi_stat_add(&stats->huge, &src->huge, 1); + mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); mi_stat_counter_add(&stats->searches, &src->searches, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { @@ -172,10 +172,15 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t } static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, FILE* out ) { - double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); - _mi_fprintf(out,"%10s: %7.1f avg\n", msg, avg); + _mi_fprintf(out, "%10s:", msg); + mi_print_amount(stat->total, -1, out); + _mi_fprintf(out, "\n"); } +static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, FILE* out) { + double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); + _mi_fprintf(out, "%10s: %7.1f avg\n", msg, avg); +} static void mi_print_header( FILE* out ) { @@ -229,15 +234,15 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n mi_stat_print(&stats->page_committed, "touched", 1, out); mi_stat_print(&stats->segments, "segments", -1, out); mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out); + mi_stat_print(&stats->segments_cache, "-cached", -1, out); mi_stat_print(&stats->pages, "pages", -1, out); mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out); mi_stat_print(&stats->pages_extended, "-extended", 0, out); + mi_stat_counter_print(&stats->page_no_retire, "-noretire", out); mi_stat_print(&stats->mmap_calls, "mmaps", 0, out); - mi_stat_print(&stats->mmap_right_align, "mmap fast", 0, out); - mi_stat_print(&stats->mmap_ensure_aligned, "mmap slow", 0, out); mi_stat_print(&stats->commit_calls, "commits", 0, out); mi_stat_print(&stats->threads, "threads", 0, out); - mi_stat_counter_print(&stats->searches, "searches", out); + mi_stat_counter_print_avg(&stats->searches, "searches", out); if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs);