From c3528203b539014ae1a2c92c0915a18bc7a7b13c Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 7 Jul 2019 18:11:21 -0700 Subject: [PATCH] fix compilation with C++, fix overrides in C++ to adhere to the spec (issue #26) --- CMakeLists.txt | 4 ++- include/mimalloc-types.h | 2 +- include/mimalloc.h | 19 ++++++++---- src/alloc-override.c | 42 +++++++++++++++++++++----- src/alloc.c | 64 ++++++++++++++++++++++++++++++++++++++-- src/heap.c | 6 ++-- src/init.c | 2 +- src/page.c | 4 +-- test/main-override.cpp | 7 +++-- test/test-api.c | 18 +++++------ 10 files changed, 133 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79d297df..66fad6d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,8 @@ cmake_minimum_required(VERSION 3.0) -project(libmimalloc C) +project(libmimalloc C CXX) include("cmake/mimalloc-config-version.cmake") set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) option(MI_OVERRIDE "Override the standard malloc interface" ON) option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" ON) @@ -76,6 +77,7 @@ endif() if(MI_USE_CXX MATCHES "ON") message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c PROPERTIES LANGUAGE CXX ) endif() # Compiler flags diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 73cd1c0a..145ea9f0 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -130,7 +130,7 @@ typedef union mi_page_flags_u { // Thread free list. // We use 2 bits of the pointer for the `use_delayed_free` and `delayed_freeing` flags. typedef union mi_thread_free_u { - uintptr_t value; + volatile uintptr_t value; struct { mi_delayed_t delayed:2; #if MI_INTPTR_SIZE==8 diff --git a/include/mimalloc.h b/include/mimalloc.h index 192a0cfb..8a6de7f1 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -233,10 +233,9 @@ mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); -// ------------------------------------------------------ -// mi prefixed implementations of various posix, unix, -// and C++ allocation functions. -// ------------------------------------------------------ +// ---------------------------------------------------------------------------------- +// mi prefixed implementations of various posix, unix, and C++ allocation functions. +// ----------------------------------------------------------------------------------- mi_decl_export size_t mi_malloc_size(void* p) mi_attr_noexcept; mi_decl_export size_t mi_malloc_usable_size(void *p) mi_attr_noexcept; @@ -251,9 +250,19 @@ mi_decl_export mi_decl_allocator void* mi_pvalloc(size_t size) mi_attr_noexcept mi_decl_export mi_decl_allocator void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_export mi_decl_allocator void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); - +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; +mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; #ifdef __cplusplus } #endif +#ifdef __cplusplus +mi_decl_export void* mi_decl_allocator mi_new(std::size_t n) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1); +#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) +#include +mi_decl_export void* mi_decl_allocator mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1); +#endif +#endif + #endif diff --git a/src/alloc-override.c b/src/alloc-override.c index 1cae7cfb..6238aafa 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -30,12 +30,14 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) #define MI_FORWARD0(fun,x) MI_FORWARD(fun) + #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) #else // use forwarding by calling our `mi_` function #define MI_FORWARD1(fun,x) { return fun(x); } #define MI_FORWARD2(fun,x,y) { return fun(x,y); } #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } #define MI_FORWARD0(fun,x) { fun(x); } + #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif #if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) @@ -81,17 +83,42 @@ terms of the MIT license. A copy of the license can be found in the file #include void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_malloc,n); - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_malloc,n); - #if (__cplusplus >= 201703L) - void* operator new( std::size_t n, std::align_val_t align) noexcept(false) MI_FORWARD2(mi_malloc_aligned,n,align); - void* operator new[]( std::size_t n, std::align_val_t align) noexcept(false) MI_FORWARD2(mi_malloc_aligned,n,align); + void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } + void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } + + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n); + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n); + + #if (__cplusplus >= 201402L) + void operator delete (void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz); + void operator delete[](void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz); + #endif + + #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) + void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete (void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast(al)); }; + void operator delete[](void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast(al)); }; + + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast(al)); } #endif + #else // ------------------------------------------------------ - // With a C compiler we override the new/delete operators - // by defining the mangled C++ names of the operators (as + // With a C compiler we cannot override the new/delete operators + // as the standard requires calling into `get_new_handler` and/or + // throwing C++ exceptions (and we cannot do that from C). So, we + // hope the standard new uses `malloc` internally which will be + // redirected anyways. + // ------------------------------------------------------ + + #if 0 + // ------------------------------------------------------ + // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). // See // ------------------------------------------------------ @@ -110,6 +137,7 @@ terms of the MIT license. A copy of the license can be found in the file #else #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif + #endif #endif // __cplusplus diff --git a/src/alloc.c b/src/alloc.c index e8543a96..10bc9bf6 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -111,7 +111,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc bool use_delayed; do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE); if (mi_unlikely(use_delayed)) { // unlikely: this only happens on the first concurrent free in a page that is in the full list @@ -143,7 +143,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc // and reset the MI_DELAYED_FREEING flag do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; tfreex.delayed = MI_NO_DELAYED_FREE; } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value)); } @@ -282,8 +282,11 @@ size_t mi_usable_size(void* p) mi_attr_noexcept { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc, - (void*)&mi_malloc_small, (void*)&mi_malloc, + (void*)&mi_malloc_small, + (void*)&mi_heap_malloc, + (void*)&mi_heap_zalloc, + (void*)&mi_heap_malloc_small }; @@ -294,6 +297,24 @@ void* _mi_externs[] = { // Allocation extensions // ------------------------------------------------------ +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + UNUSED_RELEASE(size); + mi_assert(size <= mi_usable_size(p)); + mi_free(p); +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free(p); +} + extern inline void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count,size,&total)) return NULL; @@ -444,3 +465,40 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name); } + + +#ifdef __cplusplus +#include + +static mi_decl_noinline void* mi_new_try(std::size_t n) noexcept(false) { + void* p; + do { + std::new_handler h = std::get_new_handler(); + if (h==NULL) throw std::bad_alloc(); + h(); + // and try again + p = mi_malloc(n); + } while (p==NULL); + return p; +} + +// spit out `new_try` for better assembly code +void* mi_new(std::size_t n) noexcept(false) { + void* p = mi_malloc(n); + if (mi_likely(p != NULL)) return p; + else return mi_new_try(n); +} + +// for aligned allocation its fine as it is not inlined anyways +void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) { + void* p; + while ((p = mi_malloc_aligned(n,static_cast(alignment))) == NULL) { + std::new_handler h = std::get_new_handler(); + if (h==NULL) throw std::bad_alloc(); + h(); + // and try again + }; + return p; +} + +#endif diff --git a/src/heap.c b/src/heap.c index 57cb4f7d..93968713 100644 --- a/src/heap.c +++ b/src/heap.c @@ -84,7 +84,7 @@ typedef enum mi_collect_e { static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { UNUSED(arg2); UNUSED(heap); - mi_collect_t collect = (mi_collect_t)arg_collect; + mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page); if (mi_page_all_free(page)) { // no more used blocks, free the page. TODO: should we retire here and be less aggressive? @@ -131,7 +131,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) _mi_heap_delayed_free(heap); // collect all pages owned by this thread - mi_heap_visit_pages(heap, &mi_heap_page_collect, (void*)(collect), NULL); + mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != ABANDON || heap->thread_delayed_free == NULL ); // collect segment caches @@ -480,7 +480,7 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa // Visit all heap pages as areas static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) { if (visitor == NULL) return false; - return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, visitor, arg); + return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, (void*)(visitor), arg); // note: function pointer to void* :-{ } // Just to pass arguments diff --git a/src/init.c b/src/init.c index a9671287..0ac7388d 100644 --- a/src/init.c +++ b/src/init.c @@ -426,7 +426,7 @@ static void mi_process_done(void) { // C++: use static initialization to detect process start static bool _mi_process_init(void) { mi_process_init(); - return (mi_main_thread_id != 0); + return (_mi_heap_main.thread_id != 0); } static bool mi_initialized = _mi_process_init(); diff --git a/src/page.c b/src/page.c index ea86c138..319981e5 100644 --- a/src/page.c +++ b/src/page.c @@ -114,7 +114,7 @@ void _mi_page_use_delayed_free(mi_page_t* page, bool enable) { mi_thread_free_t tfreex; do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; tfreex.delayed = (enable ? MI_USE_DELAYED_FREE : MI_NO_DELAYED_FREE); if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) { mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. @@ -140,7 +140,7 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_thread_free_t tfree; mi_thread_free_t tfreex; do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; head = (mi_block_t*)((uintptr_t)tfree.head << MI_TF_PTR_SHIFT); tfreex.head = 0; } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value)); diff --git a/test/main-override.cpp b/test/main-override.cpp index 4c22fbd7..ee3d22ab 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -36,10 +36,11 @@ int main() { free(s); Test* t = new Test(42); delete t; - int err = mi_posix_memalign(&p1,32,60); - if (!err) free(p); + int err = mi_posix_memalign(&p1,32,60); + if (!err) free(p1); + free(p); mi_collect(true); - mi_stats_print(NULL); // MIMALLOC_VERBOSE env is set to 2 + mi_stats_print(NULL); // MIMALLOC_VERBOSE env is set to 2 return 0; } diff --git a/test/test-api.c b/test/test-api.c index c06d3245..f7d9d80c 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -87,33 +87,33 @@ int main() { // --------------------------------------------------- #if defined(MI_MALLOC_OVERRIDE) && !defined(_WIN32) CHECK_BODY("posix_memalign1", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, sizeof(void*), 32); - mi_assert((err==0 && (uintptr_t)p % sizeof(void*) == 0) || p==&main); + mi_assert((err==0 && (uintptr_t)p % sizeof(void*) == 0) || p==&p); mi_free(p); result = (err==0); }); CHECK_BODY("posix_memalign_no_align", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, 3, 32); - mi_assert(p==&main); + mi_assert(p==&p); result = (err==EINVAL); }); CHECK_BODY("posix_memalign_zero", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, sizeof(void*), 0); mi_free(p); result = (err==0); }); CHECK_BODY("posix_memalign_nopow2", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, 3*sizeof(void*), 32); - result = (err==EINVAL && p==&main); + result = (err==EINVAL && p==&p); }); CHECK_BODY("posix_memalign_nomem", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, sizeof(void*), SIZE_MAX); - result = (err==ENOMEM && p==&main); + result = (err==ENOMEM && p==&p); }); #endif