enable non eager commit

include/mimalloc-types.h

@@ -166,6 +166,7 @@ typedef struct mi_page_s {
   uint8_t               segment_idx;       // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
   bool                  segment_in_use:1;  // `true` if the segment allocated this page
   bool                  is_reset:1;        // `true` if the page memory was reset
+  bool                  is_committed:1;    // `true` if the page virtual memory is committed
 
   // layout like this to optimize access in `mi_malloc` and `mi_free`
   mi_page_flags_t       flags;

src/init.c

@@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 // Empty page used to initialize the small free pages array
 const mi_page_t _mi_page_empty = {
-  0, false, false, {0},
+  0, false, false, false, {0},
   0, 0,
   NULL, 0, 0,   // free, used, cookie
   NULL, 0, {0},

src/options.c

@@ -34,7 +34,7 @@ typedef struct mi_option_desc_s {
 static mi_option_desc_t options[_mi_option_last] = {
   { 0, UNINIT, "page_reset" },
   { 0, UNINIT, "cache_reset" },
-  { 0, UNINIT, "eager_commit" },     
+  { 1, UNINIT, "eager_commit" },     
   { 0, UNINIT, "eager_region_commit" },     
   { 0, UNINIT, "large_os_pages" },   // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
   { 0, UNINIT, "reset_decommits" },

src/os.c

@@ -391,8 +391,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
   return mi_os_page_align_areax(true, addr, size, newsize);
 }
 
-// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative.
-static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) {
+// Commit/Decommit memory. 
+// Usuelly commit is aligned liberal, while decommit is aligned conservative.
+// (but not for the reset version where we want commit to be conservative as well)
+static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, mi_stats_t* stats) {
   // page align in the range, commit liberally, decommit conservative
   size_t csize;
   void* start = mi_os_page_align_areax(!commit, addr, size, &csize);
@@ -426,13 +428,18 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stat
 }
 
 bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) {
-  return mi_os_commitx(addr, size, true, stats);
+  return mi_os_commitx(addr, size, true, false /* conservative? */, stats);
 }
 
 bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
-  return mi_os_commitx(addr, size, false, stats);
+  return mi_os_commitx(addr, size, false, true /* conservative? */, stats);
 }
 
+bool _mi_os_commit_unreset(void* addr, size_t size, mi_stats_t* stats) {
+  return mi_os_commitx(addr, size, true, true /* conservative? */, stats);
+}
+
+
 // Signal to the OS that the address range is no longer in use
 // but may be used later again. This will release physical memory
 // pages and reduce swapping while keeping the memory committed.
@@ -446,6 +453,10 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
         else _mi_stat_decrease(&stats->reset, csize);
   if (!reset) return true; // nothing to do on unreset!
 
+  #if MI_DEBUG>1
+  memset(start, 0, csize); // pretend it is eagerly reset
+  #endif
+
 #if defined(_WIN32)
   // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
   // (but this is for an access pattern that immediately reuses the memory)
@@ -465,7 +476,6 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
   DWORD ok = VirtualUnlock(start, csize);
   if (ok != 0) return false;
   */
-  return true;
 #else
 #if defined(MADV_FREE)
   static int advice = MADV_FREE;
@@ -482,8 +492,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
     _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno);
   }
   //mi_assert(err == 0);
-  return (err == 0);
+  if (err != 0) return false;
 #endif
+  return true;
 }
 
 // Signal to the OS that the address range is no longer in use
@@ -501,7 +512,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
 
 bool _mi_os_unreset(void* addr, size_t size, mi_stats_t* stats) {
   if (mi_option_is_enabled(mi_option_reset_decommits)) {
-    return _mi_os_commit(addr, size, stats);  // re-commit it
+    return _mi_os_commit_unreset(addr, size, stats);  // re-commit it (conservatively!)
   }
   else {
     return mi_os_resetx(addr, size, false, stats);

src/segment.c

@@ -202,16 +202,11 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
 #define MI_SEGMENT_CACHE_MAX      (4)
 #define MI_SEGMENT_CACHE_FRACTION (8)
 
+// note: returned segment may be partially reset
 static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t* tld) {
   if (segment_size != 0 && segment_size != MI_SEGMENT_SIZE) return NULL;
   mi_segment_t* segment = tld->cache;
   if (segment == NULL) return NULL;
-  if (mi_option_is_enabled(mi_option_eager_commit) &&
-      (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)))
-  {
-    // ensure the memory is available 
-    _mi_mem_unreset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats);
-  }
   tld->cache_count--;
   tld->cache = segment->next;
   segment->next = NULL;
@@ -298,9 +293,16 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
         protection_still_good = true; // otherwise, the guard pages are still in place
       }
     }
-    if (page_kind != MI_PAGE_SMALL && !mi_option_is_enabled(mi_option_eager_commit) &&
-        (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) {
-      _mi_mem_commit(segment, segment->segment_size, tld->stats);
+    if (!mi_option_is_enabled(mi_option_eager_commit)) {
+      if (page_kind != MI_PAGE_SMALL) {
+        _mi_mem_commit(segment, segment->segment_size, tld->stats);
+      }
+      else {
+        // ok, commit (and unreset) on demand again
+      }
+    }
+    else if (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)) {
+      _mi_mem_unreset(segment, segment->segment_size, tld->stats);
     }
   }
   else {
@@ -349,7 +351,8 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
   segment->cookie = _mi_ptr_cookie(segment);
   for (uint8_t i = 0; i < segment->capacity; i++) {
     segment->pages[i].segment_idx = i;
-    segment->pages[i].is_reset = !commit;
+    segment->pages[i].is_reset = false;
+    segment->pages[i].is_committed = commit;
   }
   _mi_stat_increase(&tld->stats->page_committed, segment->segment_info_size);
   //fprintf(stderr,"mimalloc: alloc segment at %p\n", (void*)segment);
@@ -421,18 +424,18 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats)
   for (size_t i = 0; i < segment->capacity; i++) {
     mi_page_t* page = &segment->pages[i];
     if (!page->segment_in_use) {
-      if (page->is_reset) {
+      if (page->is_reset || !page->is_committed) {
         size_t psize;
         uint8_t* start = _mi_page_start(segment, page, &psize);
-        page->is_reset = false;
-        if (mi_option_is_enabled(mi_option_eager_commit)) {
-          _mi_mem_unreset(start, psize, stats);
-        } 
-        else {
-          // note we could allow both lazy commit, and page level reset if we add a `is_commit` flag...
-          // for now we use commit for both
-          _mi_mem_commit(start, psize, stats);
+        mi_assert_internal(!(page->is_reset && !page->is_committed));
+        if (!page->is_committed) {
+          page->is_committed = true;
+          _mi_mem_commit(start,psize,stats);
         }
+        if (page->is_reset) {
+          page->is_reset = false;
+          _mi_mem_unreset(start, psize, stats);
+        }         
       }
       return page;
     }
@@ -452,6 +455,7 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta
   UNUSED(stats);
   mi_assert_internal(page->segment_in_use);
   mi_assert_internal(mi_page_all_free(page));
+  mi_assert_internal(page->is_committed);
   size_t inuse = page->capacity * page->block_size;
   _mi_stat_decrease(&stats->page_committed, inuse);
   _mi_stat_decrease(&stats->pages, 1);
@@ -467,10 +471,12 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta
   // zero the page data
   uint8_t idx = page->segment_idx; // don't clear the index
   bool is_reset = page->is_reset;  // don't clear the reset flag
+  bool is_committed = page->is_committed; // don't clear the commit flag
   memset(page, 0, sizeof(*page));
   page->segment_idx = idx;
   page->segment_in_use = false;
   page->is_reset = is_reset;
+  page->is_committed = is_committed;
   segment->used--;
 }