Merge branch 'dev' into xmalloc

CMakeLists.txt

@@ -16,6 +16,7 @@ option(MI_BUILD_TESTS       "Build test executables" ON)
 option(MI_CHECK_FULL        "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF)
 option(MI_PADDING           "Enable padding to detect heap block overflow (only in debug mode)" ON)
 option(MI_XMALLOC           "Enable abort() call on memory allocation failure by default" OFF)
+option(MI_SHOW_ERRORS       "Show error and warning messages by default" OFF)
 
 include("cmake/mimalloc-config-version.cmake")
 
@@ -68,6 +69,7 @@ if(MI_OVERRIDE MATCHES "ON")
       # use zone's on macOS
       message(STATUS "  Use malloc zone to override malloc (MI_OSX_ZONE=ON)")
       list(APPEND mi_sources src/alloc-override-osx.c)
+      list(APPEND mi_defines MI_OSX_ZONE=1)
       if(NOT MI_INTERPOSE MATCHES "ON")
         message(STATUS "  (enabling INTERPOSE as well since zone's require this)")
         set(MI_INTERPOSE "ON")
@@ -111,6 +113,13 @@ if(MI_XMALLOC MATCHES "ON")
   list(APPEND mi_defines MI_XMALLOC=1)
 endif()
 
+if(MI_SHOW_ERRORS MATCHES "ON")
+  message(STATUS "Enable printing of error and warning messages by default (MI_SHOW_ERRORS=ON)")
+  list(APPEND mi_defines MI_SHOW_ERRORS=1)
+else()
+  list(APPEND mi_defines MI_SHOW_ERRORS=0)
+endif()
+
 if(MI_USE_CXX MATCHES "ON")
   message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)")
   set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX )

azure-pipelines.yml

@@ -40,8 +40,8 @@ jobs:
       cd $(BuildType)
       ctest
     displayName: CTest
-  - upload: $(Build.SourcesDirectory)/$(BuildType)
+#  - upload: $(Build.SourcesDirectory)/$(BuildType)
-    artifact: mimalloc-windows-$(BuildType)
+#    artifact: mimalloc-windows-$(BuildType)
 
 - job:
   displayName: Linux
@@ -99,8 +99,8 @@ jobs:
     displayName: Make
   - script: make test -C $(BuildType)
     displayName: CTest
-  - upload: $(Build.SourcesDirectory)/$(BuildType)
+#  - upload: $(Build.SourcesDirectory)/$(BuildType)
-    artifact: mimalloc-ubuntu-$(BuildType)
+#    artifact: mimalloc-ubuntu-$(BuildType)
 
 - job:
   displayName: macOS
@@ -127,5 +127,5 @@ jobs:
     displayName: Make
   - script: make test -C $(BuildType)
     displayName: CTest
-  - upload: $(Build.SourcesDirectory)/$(BuildType)
+#  - upload: $(Build.SourcesDirectory)/$(BuildType)
-    artifact: mimalloc-macos-$(BuildType)
+#    artifact: mimalloc-macos-$(BuildType)

src/alloc-override-osx.c

@@ -41,8 +41,11 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im
 ------------------------------------------------------ */
 
 static size_t zone_size(malloc_zone_t* zone, const void* p) {
-  UNUSED(zone); UNUSED(p);
+  UNUSED(zone);
-  return 0; // as we cannot guarantee that `p` comes from us, just return 0
+  if (!mi_is_in_heap_region(p))
+    return 0; // not our pointer, bail out
+  
+  return mi_usable_size(p);
 }
 
 static void* zone_malloc(malloc_zone_t* zone, size_t size) {

src/alloc-override.c

@@ -165,7 +165,7 @@ extern "C" {
 
 void   cfree(void* p)                    MI_FORWARD0(mi_free, p);
 void*  reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize);
-size_t malloc_size(void* p)              MI_FORWARD1(mi_usable_size,p);
+size_t malloc_size(const void* p)        MI_FORWARD1(mi_usable_size,p);
 #if !defined(__ANDROID__)
 size_t malloc_usable_size(void *p)       MI_FORWARD1(mi_usable_size,p);
 #else

src/arena.c

@@ -36,6 +36,7 @@ of 256MiB in practice.
 
 // os.c
 void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld);
+void  _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats);
 void  _mi_os_free(void* p, size_t size, mi_stats_t* stats);
 
 void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize);
@@ -213,13 +214,13 @@ void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, siz
   Arena free
 ----------------------------------------------------------- */
 
-void _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats) {
+void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats) {
   mi_assert_internal(size > 0 && stats != NULL);
   if (p==NULL) return;
   if (size==0) return;
   if (memid == MI_MEMID_OS) {
     // was a direct OS allocation, pass through
-    _mi_os_free(p, size, stats);
+    _mi_os_free_ex(p, size, all_committed, stats);
   }
   else {
     // allocated in an arena

src/options.c

@@ -51,7 +51,7 @@ typedef struct mi_option_desc_s {
 static mi_option_desc_t options[_mi_option_last] =
 {
   // stable options
-  { MI_DEBUG, UNINIT, MI_OPTION(show_errors) },
+  { MI_DEBUG | MI_SHOW_ERRORS, UNINIT, MI_OPTION(show_errors) },
   { 0, UNINIT, MI_OPTION(show_stats) },
   { 0, UNINIT, MI_OPTION(verbose) },
 
@@ -260,13 +260,17 @@ static void mi_recurse_exit(void) {
 }
 
 void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
-  if (!mi_recurse_enter()) return;
   if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) { // TODO: use mi_out_stderr for stderr?
+    if (!mi_recurse_enter()) return;
     out = mi_out_get_default(&arg);
+    if (prefix != NULL) out(prefix, arg);
+    out(message, arg);
+    mi_recurse_exit();
+  }
+  else {
+    if (prefix != NULL) out(prefix, arg);
+    out(message, arg);
   }
-  if (prefix != NULL) out(prefix,arg);
-  out(message,arg);
-  mi_recurse_exit();
 }
 
 // Define our own limited `fprintf` that avoids memory allocation.

src/region.c

@@ -49,7 +49,7 @@ bool    _mi_os_reset(void* p, size_t size, mi_stats_t* stats);
 bool    _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
 
 // arena.c
-void    _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats);
+void    _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats);
 void*   _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
 void*   _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
 
@@ -187,7 +187,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large,
   const uintptr_t idx = mi_atomic_increment(&regions_count);
   if (idx >= MI_REGION_MAX) {
     mi_atomic_decrement(&regions_count);
-    _mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats);
+    _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats);
     _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, GiB));
     return false;
   }
@@ -391,7 +391,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re
   mem_region_t* region;
   if (mi_memid_is_arena(id,&region,&bit_idx,&arena_memid)) {
    // was a direct arena allocation, pass through
-    _mi_arena_free(p, size, arena_memid, tld->stats);
+    _mi_arena_free(p, size, arena_memid, full_commit, tld->stats);
   }
   else {
     // allocated in a region
@@ -454,12 +454,13 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
         // on success, free the whole region
         uint8_t* start = mi_atomic_read_ptr(uint8_t,&regions[i].start);
         size_t arena_memid = mi_atomic_read_relaxed(&regions[i].arena_memid);
+        uintptr_t commit = mi_atomic_read_relaxed(&regions[i].commit);
         memset(&regions[i], 0, sizeof(mem_region_t));
         // and release the whole region
         mi_atomic_write(&region->info, 0);
         if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) {         
           _mi_abandoned_await_readers(); // ensure no pending reads
-          _mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats);
+          _mi_arena_free(start, MI_REGION_SIZE, arena_memid, (~commit == 0), tld->stats);
         }
       }
     }

src/segment.c

@@ -204,9 +204,9 @@ static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t*
     mi_segment_protect_range((uint8_t*)segment + segment->segment_info_size - os_page_size, os_page_size, protect);
     if (MI_SECURE <= 1 || segment->capacity == 1) {
       // and protect the last (or only) page too
-      mi_assert_internal(segment->page_kind >= MI_PAGE_LARGE);
+      mi_assert_internal(MI_SECURE <= 1 || segment->page_kind >= MI_PAGE_LARGE);
       uint8_t* start = (uint8_t*)segment + segment->segment_size - os_page_size;
-      if (protect && !mi_option_is_enabled(mi_option_eager_page_commit)) {
+      if (protect && !segment->mem_is_committed) {
         // ensure secure page is committed
         _mi_mem_commit(start, os_page_size, NULL, tld);
       }
@@ -236,12 +236,12 @@ static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, m
   void* start = mi_segment_raw_page_start(segment, page, &psize);
   page->is_reset = true;
   mi_assert_internal(size <= psize);
-  size_t reset_size = (size == 0 || size > psize ? psize : size);
+  size_t reset_size = ((size == 0 || size > psize) ? psize : size);
   if (size == 0 && segment->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) {
     mi_assert_internal(page->xblock_size > 0);
     reset_size = page->capacity * mi_page_block_size(page);
   }
-  _mi_mem_reset(start, reset_size, tld->os);
+  if (reset_size > 0) _mi_mem_reset(start, reset_size, tld->os);
 }
 
 static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld)
@@ -258,7 +258,7 @@ static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size,
     unreset_size = page->capacity * mi_page_block_size(page);
   }
   bool is_zero = false;
-  _mi_mem_unreset(start, unreset_size, &is_zero, tld->os);
+  if (unreset_size > 0) _mi_mem_unreset(start, unreset_size, &is_zero, tld->os);
   if (is_zero) page->is_zero_init = true;
 }
 
@@ -627,8 +627,13 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
     if (!commit) {
       // ensure the initial info is committed
       bool commit_zero = false;
-      _mi_mem_commit(segment, pre_size, &commit_zero, tld->os);
+      bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld->os);
       if (commit_zero) is_zero = true;
+      if (!ok) {
+        // commit failed; we cannot touch the memory: free the segment directly and return `NULL`
+        _mi_mem_free(segment, MI_SEGMENT_SIZE, memid, false, false, os_tld);
+        return NULL;  
+      }
     }
     segment->memid = memid;
     segment->mem_is_fixed = mem_large;
@@ -714,28 +719,30 @@ static bool mi_segment_has_free(const mi_segment_t* segment) {
   return (segment->used < segment->capacity);
 }
 
-static void mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) {
+static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) {
   mi_assert_internal(_mi_page_segment(page) == segment);
   mi_assert_internal(!page->segment_in_use);
-  // set in-use before doing unreset to prevent delayed reset
   mi_pages_reset_remove(page, tld);
-  page->segment_in_use = true;
+  // check commit
-  segment->used++;
   if (!page->is_committed) {
     mi_assert_internal(!segment->mem_is_fixed);
     mi_assert_internal(!page->is_reset);
-    page->is_committed = true;
+    if (segment->page_kind < MI_PAGE_LARGE || mi_option_is_enabled(mi_option_eager_page_commit)) {
-    if (segment->page_kind < MI_PAGE_LARGE
-      || !mi_option_is_enabled(mi_option_eager_page_commit)) {
       size_t psize;
       uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
       bool is_zero = false;
       const size_t gsize = (MI_SECURE >= 2 ? _mi_os_page_size() : 0);
-      _mi_mem_commit(start, psize + gsize, &is_zero, tld->os);
+      bool ok = _mi_mem_commit(start, psize + gsize, &is_zero, tld->os);
+      if (!ok) return false; // failed to commit!
       if (gsize > 0) { mi_segment_protect_range(start + psize, gsize, true); }
       if (is_zero) { page->is_zero_init = true; }
     }
+    page->is_committed = true;
   }
+  // set in-use before doing unreset to prevent delayed reset
+  page->segment_in_use = true;
+  segment->used++;
+  // check reset
   if (page->is_reset) {
     mi_page_unreset(segment, page, 0, tld); // todo: only unreset the part that was reset?
   }
@@ -746,6 +753,7 @@ static void mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg
     mi_assert_internal(!mi_segment_has_free(segment));
     mi_segment_remove_from_free_queue(segment, tld);
   }
+  return true;
 }
 
 
@@ -1212,8 +1220,8 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_segments_tld_t*
   for (size_t i = 0; i < segment->capacity; i++) {  // TODO: use a bitmap instead of search?
     mi_page_t* page = &segment->pages[i];
     if (!page->segment_in_use) {
-      mi_segment_page_claim(segment, page, tld);
+      bool ok = mi_segment_page_claim(segment, page, tld);
-      return page;
+      if (ok) return page;
     }
   }
   mi_assert(false);

src/static.c

@@ -24,5 +24,8 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "alloc.c"
 #include "alloc-aligned.c"
 #include "alloc-posix.c"
+#if MI_OSX_ZONE
+#include "alloc-override-osx.c"
+#endif
 #include "init.c"
 #include "options.c"

src/stats.c

@@ -237,9 +237,51 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin
 #endif
 
 
+
+//------------------------------------------------------------
+// Use an output wrapper for line-buffered output
+// (which is nice when using loggers etc.)
+//------------------------------------------------------------
+typedef struct buffered_s {
+  mi_output_fun* out;   // original output function
+  void*          arg;   // and state
+  char*          buf;   // local buffer of at least size `count+1`
+  size_t         used;  // currently used chars `used <= count`  
+  size_t         count; // total chars available for output
+} buffered_t;
+
+static void mi_buffered_flush(buffered_t* buf) {
+  buf->buf[buf->used] = 0;
+  _mi_fputs(buf->out, buf->arg, NULL, buf->buf);
+  buf->used = 0;
+}
+
+static void mi_buffered_out(const char* msg, void* arg) {
+  buffered_t* buf = (buffered_t*)arg;
+  if (msg==NULL || buf==NULL) return;
+  for (const char* src = msg; *src != 0; src++) {
+    char c = *src;
+    if (buf->used >= buf->count) mi_buffered_flush(buf);
+    mi_assert_internal(buf->used < buf->count);
+    buf->buf[buf->used++] = c;
+    if (c == '\n') mi_buffered_flush(buf);
+  }
+}
+
+//------------------------------------------------------------
+// Print statistics
+//------------------------------------------------------------
+
 static void mi_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit);
 
-static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out, void* arg) mi_attr_noexcept {
+static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out0, void* arg0) mi_attr_noexcept {
+  // wrap the output function to be line buffered
+  char buf[256];
+  buffered_t buffer = { out0, arg0, buf, 0, 255 };
+  mi_output_fun* out = &mi_buffered_out;
+  void* arg = &buffer;
+
+  // and print using that
   mi_print_header(out,arg);
   #if MI_STAT>1
   mi_stat_count_t normal = { 0,0,0,0 };
@@ -287,7 +329,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun
     _mi_fprintf(out, arg, ", commit charge: ");
     mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");
   }
-  _mi_fprintf(out, arg, "\n");
+  _mi_fprintf(out, arg, "\n");  
 }
 
 static mi_msecs_t mi_time_start; // = 0