more complete C++ support for compliance and performance

include/mimalloc.h

@@ -259,17 +259,15 @@ mi_decl_export mi_decl_allocator void* mi_reallocarray(void* p, size_t count, si
 mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
 mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
 mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;
+
+mi_decl_export void* mi_new(size_t n) mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_export void* mi_new_aligned(size_t n, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_export void* mi_new_nothrow(size_t n) mi_attr_malloc mi_attr_alloc_size(1);
+mi_decl_export void* mi_new_aligned_nothrow(size_t n, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1);
+
 #ifdef __cplusplus
 }
 #endif
 
-#ifdef __cplusplus
-#include <cstddef>
-mi_decl_export void* mi_new(std::size_t n) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1);
-#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
-#include <new>
-mi_decl_export void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1);
-#endif
-#endif
 
 #endif

src/alloc-override.c

@@ -88,60 +88,66 @@ terms of the MIT license. A copy of the license can be found in the file
   void operator delete(void* p) noexcept              MI_FORWARD0(mi_free,p);
   void operator delete[](void* p) noexcept            MI_FORWARD0(mi_free,p);
 
-  void* operator new(std::size_t n) noexcept(false)   { return mi_new(n); }
+  void* operator new(std::size_t n) noexcept(false)   MI_FORWARD1(mi_new,n);
-  void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
+  void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n);
 
-  void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n);
+  void* operator new  (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); }
-  void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n);
+  void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); }
 
   #if (__cplusplus >= 201402L)
-  void operator delete  (void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz);
+  void operator delete  (void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n);
-  void operator delete[](void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz);
+  void operator delete[](void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n);
   #endif
 
   #if (__cplusplus > 201402L || defined(__cpp_aligned_new))
   void operator delete  (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
   void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
-  void operator delete  (void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast<size_t>(al)); };
+  void operator delete  (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
-  void operator delete[](void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast<size_t>(al)); };
+  void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
 
-  void* operator new( std::size_t n, std::align_val_t al)   noexcept(false) { return mi_new_aligned(n,al); }
+  void* operator new( std::size_t n, std::align_val_t al)   noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); }
+  void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast<size_t>(al)); }
-  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast<size_t>(al)); }
+  void* operator new  (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
-  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast<size_t>(al)); }
+  void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast<size_t>(al)); }
   #endif
 
-#else
+#elif (defined(__GNUC__) || defined(__clang__))
-  // ------------------------------------------------------
-  // With a C compiler we cannot override the new/delete operators
-  // as the standard requires calling into `get_new_handler` and/or
-  // throwing C++ exceptions (and we cannot do that from C). So, we
-  // hope the standard new uses `malloc` internally which will be
-  // redirected anyways.
-  // ------------------------------------------------------
-
-  #if 0
   // ------------------------------------------------------
   // Override by defining the mangled C++ names of the operators (as
   // used by GCC and CLang).
   // See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
   // ------------------------------------------------------
-  void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete
+  void _ZdlPv(void* p)            MI_FORWARD0(mi_free,p); // delete
-  void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[]
+  void _ZdaPv(void* p)            MI_FORWARD0(mi_free,p); // delete[]
+  void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n);
+  void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n);
+  void _ZdlPvSt11align_val_t(void* p, size_t al)            { mi_free_aligned(p,al); }
+  void _ZdaPvSt11align_val_t(void* p, size_t al)            { mi_free_aligned(p,al); }
+  void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
+  void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
+
+  typedef struct mi_nothrow_s {  } mi_nothrow_t;
   #if (MI_INTPTR_SIZE==8)
-    void* _Znwm(uint64_t n)                  MI_FORWARD1(mi_malloc,n);               // new 64-bit
+    void* _Znwm(size_t n)                             MI_FORWARD1(mi_new,n);  // new 64-bit
-    void* _Znam(uint64_t n)                  MI_FORWARD1(mi_malloc,n);               // new[] 64-bit
+    void* _Znam(size_t n)                             MI_FORWARD1(mi_new,n);  // new[] 64-bit
-    void* _Znwmm(uint64_t n, uint64_t align) { return mi_malloc_aligned(n,align); } // aligned new 64-bit
+    void* _ZnwmSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al);
-    void* _Znamm(uint64_t n, uint64_t align) { return mi_malloc_aligned(n,align); }  // aligned new[] 64-bit
+    void* _ZnamSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al);
+    void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
+    void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
   #elif (MI_INTPTR_SIZE==4)
-    void* _Znwj(uint32_t n)                  MI_FORWARD1(mi_malloc,n);               // new 32-bit
+    void* _Znwj(size_t n)                             MI_FORWARD1(mi_new,n);  // new 64-bit
-    void* _Znaj(uint32_t n)                  MI_FORWARD1(mi_malloc,n);               // new[] 32-bit
+    void* _Znaj(size_t n)                             MI_FORWARD1(mi_new,n);  // new[] 64-bit
-    void* _Znwjj(uint32_t n, uint32_t align) { return mi_malloc_aligned(n,align); }  // aligned new 32-bit
+    void* _ZnwjSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al);
-    void* _Znajj(uint32_t n, uint32_t align) { return mi_malloc_aligned(n,align); }  // aligned new[] 32-bit
+    void* _ZnajSt11align_val_t(size_t n, size_t al)   MI_FORWARD2(mi_new_aligned, n, al);
+    void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
+    void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
+    void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
   #else
   #error "define overloads for new/delete for this platform (just for performance, can be skipped)"
   #endif
-  #endif
 #endif // __cplusplus
 
 

src/alloc.c

@@ -256,8 +256,8 @@ bool _mi_free_delayed_block(mi_block_t* block) {
   mi_assert_internal(_mi_thread_id() == segment->thread_id);
   mi_page_t* page = _mi_segment_page_of(segment, block);
   if (mi_tf_delayed(page->thread_free) == MI_DELAYED_FREEING) {
-    // we might already start delayed freeing while another thread has not yet 
+    // we might already start delayed freeing while another thread has not yet
-    // reset the delayed_freeing flag; in that case don't free it quite yet if 
+    // reset the delayed_freeing flag; in that case don't free it quite yet if
     // this is the last block remaining.
     if (page->used - page->thread_freed == 1) return false;
   }
@@ -282,9 +282,6 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept {
 }
 
 
-
-
-
 // ------------------------------------------------------
 // ensure explicit external inline definitions are emitted!
 // ------------------------------------------------------
@@ -298,8 +295,6 @@ void* _mi_externs[] = {
   (void*)&mi_heap_zalloc,
   (void*)&mi_heap_malloc_small
 };
-
-
 #endif
 
 
@@ -360,7 +355,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero)
   if (newsize <= size && newsize >= (size / 2)) {
     return p;  // reallocation still fits and not more than 50% waste
   }
-  void* newp = mi_heap_malloc(heap,newsize); 
+  void* newp = mi_heap_malloc(heap,newsize);
   if (mi_likely(newp != NULL)) {
     if (zero && newsize > size) {
       // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
@@ -410,6 +405,10 @@ void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
   return mi_heap_reallocf(mi_get_default_heap(),p,newsize);
 }
 
+// ------------------------------------------------------
+// strdup, strndup, and realpath
+// ------------------------------------------------------
+
 // `strdup` using mi_malloc
 char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {
   if (s == NULL) return NULL;
@@ -496,41 +495,91 @@ char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {
   return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name);
 }
 
+/*-------------------------------------------------------
+C++ new and new_aligned
+The standard requires calling into `get_new_handler` and
+throwing the bad_alloc exception on failure. If we compile
+with a C++ compiler we can implement this precisely. If we
+use a C compiler we cannot throw a `bad_alloc` exception
+but we call `exit` instead (i.e. not returning).
+-------------------------------------------------------*/
 
 #ifdef __cplusplus
 #include <new>
-
+static bool mi_try_new_handler(bool nothrow) {
-static mi_decl_noinline void* mi_new_try(std::size_t n) noexcept(false) {
+  std::new_handler h = std::get_new_handler();
-  void* p;
+  if (h==NULL) {
-  do {
+    if (!nothrow) throw std::bad_alloc();
-    std::new_handler h = std::get_new_handler();
+    return false;
-    if (h==NULL) throw std::bad_alloc();
+  }
+  else {
     h();
-    // and try again
+    return true;
+  }
+}
+#else
+#include <errno.h>
+#ifndef ENOMEM
+#define ENOMEM 12
+#endif
+typedef void (*std_new_handler_t)();
+
+#if (defined(__GNUC__) || defined(__clang__))
+std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() {
+  return NULL;
+}
+std_new_handler_t mi_get_new_handler() {
+  return _ZSt15get_new_handlerv();
+}
+#else
+std_new_handler_t mi_get_new_handler() {
+  return NULL;
+}
+#endif
+
+static bool mi_try_new_handler(bool nothrow) {
+  std_new_handler_t h = mi_get_new_handler();
+  if (h==NULL) {
+    if (!nothrow) exit(ENOMEM);
+    return false;
+  }
+  else {
+    h();
+    return true;
+  }
+}
+#endif
+
+static mi_decl_noinline void* mi_try_new(size_t n, bool nothrow ) {
+  void* p = NULL;
+  while(p == NULL && mi_try_new_handler(nothrow)) {
     p = mi_malloc(n);
-  } while (p==NULL);
+  }
   return p;
 }
 
-// spit out `new_try` for better assembly code
+void* mi_new(size_t n) {
-void* mi_new(std::size_t n) noexcept(false) {
   void* p = mi_malloc(n);
-  if (mi_likely(p != NULL)) return p;
+  if (mi_unlikely(p == NULL)) return mi_try_new(n,false);
-                       else return mi_new_try(n);
-}
-
-#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
-// for aligned allocation its fine as it is not inlined anyways
-void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) {
-  void* p;
-  while ((p = mi_malloc_aligned(n,static_cast<size_t>(alignment))) == NULL) {
-    std::new_handler h = std::get_new_handler();
-    if (h==NULL) throw std::bad_alloc();
-    h();
-    // and try again    
-  };
   return p;
 }
-#endif
 
-#endif
+void* mi_new_aligned(size_t n, size_t alignment) {
+  void* p;
+  do { p = mi_malloc_aligned(n, alignment); }
+  while(p == NULL && mi_try_new_handler(false));
+  return p;
+}
+
+void* mi_new_nothrow(size_t n) {
+  void* p = mi_malloc(n);
+  if (mi_unlikely(p == NULL)) return mi_try_new(n,true);
+  return p;
+}
+
+void* mi_new_aligned_nothrow(size_t n, size_t alignment) {
+  void* p;
+  do { p = mi_malloc_aligned(n, alignment); }
+  while (p == NULL && mi_try_new_handler(true));
+  return p;
+}

test/main-override.cpp

@@ -5,6 +5,8 @@
 
 #include <mimalloc.h>
 
+#include <new>
+
 static void* p = malloc(8);
 
 void free_p() {
@@ -36,6 +38,8 @@ int main() {
   free(s);
   Test* t = new Test(42);
   delete t;
+  t = new (std::nothrow) Test(42);
+  delete t;
   int err = mi_posix_memalign(&p1,32,60);
   if (!err) free(p1);
   free(p);