improved stats

include/mimalloc-internal.h

@@ -307,7 +307,7 @@ static inline bool mi_page_all_used(mi_page_t* page) {
 static inline bool mi_page_mostly_used(const mi_page_t* page) {
   if (page==NULL) return true;
   uint16_t frac = page->reserved / 8U;
-  return (page->reserved - page->used + page->thread_freed < frac);
+  return (page->reserved - page->used + page->thread_freed <= frac);
 }
 
 static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {

include/mimalloc-types.h

@@ -324,12 +324,12 @@ typedef struct mi_stats_s {
   mi_stat_count_t pages_abandoned;
   mi_stat_count_t pages_extended;
   mi_stat_count_t mmap_calls;
-  mi_stat_count_t mmap_right_align;
-  mi_stat_count_t mmap_ensure_aligned;
   mi_stat_count_t commit_calls;
   mi_stat_count_t threads;
   mi_stat_count_t huge;
   mi_stat_count_t malloc;
+  mi_stat_count_t segments_cache;
+  mi_stat_counter_t page_no_retire;
   mi_stat_counter_t searches;
 #if MI_STAT>1
   mi_stat_count_t normal[MI_BIN_HUGE+1];

src/init.c

@@ -61,7 +61,8 @@ const mi_page_t _mi_page_empty = {
   MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
   MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
   MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
-  MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
+  MI_STAT_COUNT_NULL(), \
+  { 0, 0 }, \
   { 0, 0 } \
   MI_STAT_COUNT_END_NULL()
 

src/page.c

@@ -216,7 +216,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
   mi_page_t* page = _mi_segment_page_alloc(block_size, &heap->tld->segments, &heap->tld->os);
   if (page == NULL) return NULL;
   mi_page_init(heap, page, block_size, &heap->tld->stats);
-  mi_heap_stat_increase( heap, pages, 1);
+  _mi_stat_increase( &heap->tld->stats.pages, 1);
   mi_page_queue_push(heap, pq, page);
   mi_assert_expensive(_mi_page_is_valid(page));
   return page;
@@ -352,7 +352,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
 
   // account for huge pages here
   if (page->block_size > MI_LARGE_SIZE_MAX) {
-    mi_heap_stat_decrease(page->heap, huge, page->block_size);
+    _mi_stat_decrease(&page->heap->tld->stats.huge, page->block_size);
   }
 
   // remove from the page list
@@ -386,6 +386,7 @@ void _mi_page_retire(mi_page_t* page) {
   // if its neighbours are almost fully used.
   if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) {
     if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) {
+      _mi_stat_counter_increase(&page->heap->tld->stats.page_no_retire,1);
       return; // dont't retire after all
     }
   }
@@ -700,7 +701,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
   if (page != NULL) {
     mi_assert_internal(mi_page_immediate_available(page));
     mi_assert_internal(page->block_size == block_size);
-    mi_heap_stat_increase( heap, huge, block_size);
+    _mi_stat_increase( &heap->tld->stats.huge, block_size);
   }
   return page;
 }

src/segment.c

@@ -248,17 +248,19 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t
   tld->cache = segment->next;
   segment->next = NULL;
   mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE);
+  _mi_stat_decrease(&tld->stats->segments_cache, 1);
   return segment;
 }
 
 static bool mi_segment_cache_full(mi_segments_tld_t* tld) {
-  if (tld->cache_count <  MI_SEGMENT_CACHE_MAX &&
+  if (tld->cache_count <  MI_SEGMENT_CACHE_MAX 
-      tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { // always allow 1 element cache
+      && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))
+     ) { // always allow 1 element cache
     return false;
   }
   // take the opportunity to reduce the segment cache if it is too large (now)
   // TODO: this never happens as we check against peak usage, should we use current usage instead?
-  while (tld->cache_count > (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) {
+  while (tld->cache_count > MI_SEGMENT_CACHE_MAX ) { //(1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) {
     mi_segment_t* segment = mi_segment_cache_pop(0,tld);
     mi_assert_internal(segment != NULL);
     if (segment != NULL) mi_segment_os_free(segment, segment->segment_size, tld);
@@ -269,7 +271,9 @@ static bool mi_segment_cache_full(mi_segments_tld_t* tld) {
 static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) {
   mi_assert_internal(!mi_segment_is_in_free_queue(segment, tld));
   mi_assert_internal(segment->next == NULL);
-  if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) return false;
+  if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) {
+    return false;
+  }
   mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE);
   if (mi_option_is_enabled(mi_option_cache_reset)) {
     _mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats);
@@ -277,6 +281,7 @@ static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld)
   segment->next = tld->cache;
   tld->cache = segment;
   tld->cache_count++;
+  _mi_stat_increase(&tld->stats->segments_cache,1);
   return true;
 }
 

src/stats.c

@@ -99,14 +99,14 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
   mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1);
   mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1);
   mi_stat_add(&stats->mmap_calls, &src->mmap_calls, 1);
-  mi_stat_add(&stats->mmap_ensure_aligned, &src->mmap_ensure_aligned, 1);
-  mi_stat_add(&stats->mmap_right_align, &src->mmap_right_align, 1);
   mi_stat_add(&stats->commit_calls, &src->commit_calls, 1);
   mi_stat_add(&stats->threads, &src->threads, 1);
   mi_stat_add(&stats->pages_extended, &src->pages_extended, 1);
 
   mi_stat_add(&stats->malloc, &src->malloc, 1);
+  mi_stat_add(&stats->segments_cache, &src->segments_cache, 1);
   mi_stat_add(&stats->huge, &src->huge, 1);
+  mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1);
   mi_stat_counter_add(&stats->searches, &src->searches, 1);
 #if MI_STAT>1
   for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
@@ -172,10 +172,15 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
 }
 
 static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, FILE* out ) {
-  double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count);
+  _mi_fprintf(out, "%10s:", msg);
-  _mi_fprintf(out,"%10s: %7.1f avg\n", msg, avg);
+  mi_print_amount(stat->total, -1, out);
+  _mi_fprintf(out, "\n");
 }
 
+static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, FILE* out) {
+  double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count);
+  _mi_fprintf(out, "%10s: %7.1f avg\n", msg, avg);
+}
 
 
 static void mi_print_header( FILE* out ) {
@@ -229,15 +234,15 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n
   mi_stat_print(&stats->page_committed, "touched", 1, out);
   mi_stat_print(&stats->segments, "segments", -1, out);
   mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out);
+  mi_stat_print(&stats->segments_cache, "-cached", -1, out);
   mi_stat_print(&stats->pages, "pages", -1, out);
   mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out);
   mi_stat_print(&stats->pages_extended, "-extended", 0, out);
+  mi_stat_counter_print(&stats->page_no_retire, "-noretire", out);
   mi_stat_print(&stats->mmap_calls, "mmaps", 0, out);
-  mi_stat_print(&stats->mmap_right_align, "mmap fast", 0, out);
-  mi_stat_print(&stats->mmap_ensure_aligned, "mmap slow", 0, out);
   mi_stat_print(&stats->commit_calls, "commits", 0, out);
   mi_stat_print(&stats->threads, "threads", 0, out);
-  mi_stat_counter_print(&stats->searches, "searches", out);
+  mi_stat_counter_print_avg(&stats->searches, "searches", out);
 
   if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs);