diff --git a/CMakeLists.txt b/CMakeLists.txt index 782a7d55..9a4e3d67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,8 @@ cmake_minimum_required(VERSION 3.0) -project(libmimalloc C) +project(libmimalloc C CXX) include("cmake/mimalloc-config-version.cmake") set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) option(MI_OVERRIDE "Override the standard malloc interface" ON) option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" ON) @@ -20,6 +21,7 @@ set(mi_sources src/page.c src/alloc.c src/alloc-aligned.c + src/alloc-posix.c src/heap.c src/options.c src/init.c) @@ -76,6 +78,7 @@ endif() if(MI_USE_CXX MATCHES "ON") message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c PROPERTIES LANGUAGE CXX ) endif() # Compiler flags @@ -160,6 +163,8 @@ target_include_directories(mimalloc-obj PUBLIC $ ) +install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_dir}) + install(FILES $ DESTINATION ${mi_install_dir} RENAME ${mi_basename}${CMAKE_C_OUTPUT_EXTENSION} ) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index c6d89be0..794da054 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -212,12 +212,13 @@ - true - true - true - true + false + false + false + false + diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters index bad4bde4..d2892c32 100644 --- a/ide/vs2017/mimalloc-override.vcxproj.filters +++ b/ide/vs2017/mimalloc-override.vcxproj.filters @@ -61,5 +61,8 @@ Source Files + + Source Files + - \ No newline at end of file + diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index 675d6acb..7976af56 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -89,6 +89,7 @@ true true ..\..\include + stdcpp17 Console @@ -101,6 +102,7 @@ true true ..\..\include + stdcpp17 Console @@ -116,6 +118,7 @@ true ..\..\include _MBCS;%(PreprocessorDefinitions);NDEBUG + stdcpp17 true @@ -133,6 +136,7 @@ true ..\..\include _MBCS;%(PreprocessorDefinitions);NDEBUG + stdcpp17 true diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index c5e86acc..22208d8e 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -94,6 +94,7 @@ true ../../include MI_DEBUG=3;_MBCS;%(PreprocessorDefinitions); + Default @@ -110,6 +111,7 @@ true ../../include MI_DEBUG=3;_MBCS;%(PreprocessorDefinitions); + Default @@ -144,6 +146,7 @@ Neither false false + Default true @@ -174,6 +177,7 @@ Neither false false + Default true @@ -217,6 +221,7 @@ true true + diff --git a/ide/vs2017/mimalloc.vcxproj.filters b/ide/vs2017/mimalloc.vcxproj.filters index 7a026e74..8bdeccf9 100644 --- a/ide/vs2017/mimalloc.vcxproj.filters +++ b/ide/vs2017/mimalloc.vcxproj.filters @@ -53,6 +53,9 @@ Source Files + + Source Files + @@ -68,4 +71,4 @@ Header Files - \ No newline at end of file + diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index a8f2812c..3fe30474 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -66,33 +66,35 @@ static inline void* mi_atomic_exchange_ptr(volatile void** p, void* exchange) { #include #include #if (MI_INTPTR_SIZE==8) +typedef LONG64 msc_intptr_t; #define RC64(f) f##64 #else +typedef LONG msc_intptr_t; #define RC64(f) f #endif static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p) { - return (uintptr_t)RC64(_InterlockedIncrement)((volatile intptr_t*)p); + return (uintptr_t)RC64(_InterlockedIncrement)((volatile msc_intptr_t*)p); } static inline uint32_t mi_atomic_increment32(volatile uint32_t* p) { - return (uint32_t)_InterlockedIncrement((volatile int32_t*)p); + return (uint32_t)_InterlockedIncrement((volatile LONG*)p); } static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p) { - return (uintptr_t)RC64(_InterlockedDecrement)((volatile intptr_t*)p); + return (uintptr_t)RC64(_InterlockedDecrement)((volatile msc_intptr_t*)p); } static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub) { - return (uintptr_t)RC64(_InterlockedExchangeAdd)((volatile intptr_t*)p, -((intptr_t)sub)) - sub; + return (uintptr_t)RC64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)) - sub; } static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub) { - return (uint32_t)_InterlockedExchangeAdd((volatile int32_t*)p, -((int32_t)sub)) - sub; + return (uint32_t)_InterlockedExchangeAdd((volatile LONG*)p, -((LONG)sub)) - sub; } static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare) { - return ((int32_t)compare == _InterlockedCompareExchange((volatile int32_t*)p, (int32_t)exchange, (int32_t)compare)); + return ((int32_t)compare == _InterlockedCompareExchange((volatile LONG*)p, (LONG)exchange, (LONG)compare)); } static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare) { - return (compare == RC64(_InterlockedCompareExchange)((volatile intptr_t*)p, (intptr_t)exchange, (intptr_t)compare)); + return (compare == RC64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange, (msc_intptr_t)compare)); } static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) { - return (uintptr_t)RC64(_InterlockedExchange)((volatile intptr_t*)p, (intptr_t)exchange); + return (uintptr_t)RC64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); } static inline uintptr_t mi_atomic_read(volatile uintptr_t* p) { return *p; diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index b2526379..fa8a25cc 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -92,7 +92,7 @@ void _mi_stats_done(mi_stats_t* stats); void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic` void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero); void* _mi_realloc_zero(void* p, size_t size, bool zero); -mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, void* p); +mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); void _mi_free_delayed_block(mi_block_t* block); #if MI_DEBUG>1 diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index b25250f7..9b5e87bb 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -130,7 +130,7 @@ typedef union mi_page_flags_u { // Thread free list. // We use 2 bits of the pointer for the `use_delayed_free` and `delayed_freeing` flags. typedef union mi_thread_free_u { - uintptr_t value; + volatile uintptr_t value; struct { mi_delayed_t delayed:2; #if MI_INTPTR_SIZE==8 diff --git a/include/mimalloc.h b/include/mimalloc.h index d88e9bd1..2807bec4 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -41,6 +41,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_malloc #define mi_attr_alloc_size(s) #define mi_attr_alloc_size2(s1,s2) + #define mi_cdecl __cdecl #elif defined(__GNUC__) || defined(__clang__) #define mi_decl_thread __thread #define mi_decl_export __attribute__((visibility("default"))) @@ -51,7 +52,8 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #else #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) - #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_cdecl // leads to warnings... __attribute__((cdecl)) #endif #else #define mi_decl_thread __thread @@ -60,6 +62,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_malloc #define mi_attr_alloc_size(s) #define mi_attr_alloc_size2(s1,s2) + #define mi_cdecl #endif // ------------------------------------------------------ @@ -104,8 +107,8 @@ mi_decl_export mi_decl_allocator void* mi_reallocf(void* p, size_t newsize) mi_decl_export mi_decl_allocator void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_export mi_decl_allocator void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); -mi_decl_export size_t mi_usable_size(void* p) mi_attr_noexcept; -mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; +mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; +mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; mi_decl_export void mi_collect(bool force) mi_attr_noexcept; mi_decl_export void mi_stats_print(FILE* out) mi_attr_noexcept; @@ -186,7 +189,7 @@ typedef struct mi_heap_area_s { size_t block_size; // size in bytes of each block } mi_heap_area_t; -typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); +typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); @@ -232,8 +235,38 @@ mi_decl_export long mi_option_get(mi_option_t option); mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); + +// ---------------------------------------------------------------------------------- +// mi prefixed implementations of various posix, unix, and C++ allocation functions. +// ----------------------------------------------------------------------------------- + +mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; +mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; + +mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; +mi_decl_export int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; +mi_decl_export mi_decl_allocator void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_export mi_decl_allocator void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); + +mi_decl_export mi_decl_allocator void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_export mi_decl_allocator void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_export mi_decl_allocator void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); + +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; +mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; #ifdef __cplusplus } #endif +#ifdef __cplusplus +#include +mi_decl_export void* mi_new(std::size_t n) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1); +#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) +#include +mi_decl_export void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1); +#endif +#endif + #endif diff --git a/src/alloc-override.c b/src/alloc-override.c index 7c052690..6238aafa 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -25,15 +25,19 @@ terms of the MIT license. A copy of the license can be found in the file #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) // use aliasing to alias the exported function to one of our `mi_` functions - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))) - #define MI_FORWARD1(fun,x) MI_FORWARD(fun) - #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) - #define MI_FORWARD0(fun,x) MI_FORWARD(fun) + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))) + #define MI_FORWARD1(fun,x) MI_FORWARD(fun) + #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) + #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) + #define MI_FORWARD0(fun,x) MI_FORWARD(fun) + #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) #else // use forwarding by calling our `mi_` function - #define MI_FORWARD1(fun,x) { return fun(x); } - #define MI_FORWARD2(fun,x,y) { return fun(x,y); } - #define MI_FORWARD0(fun,x) { fun(x); } + #define MI_FORWARD1(fun,x) { return fun(x); } + #define MI_FORWARD2(fun,x,y) { return fun(x,y); } + #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } + #define MI_FORWARD0(fun,x) { fun(x); } + #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif #if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) @@ -59,9 +63,7 @@ terms of the MIT license. A copy of the license can be found in the file void* malloc(size_t size) mi_attr_noexcept MI_FORWARD1(mi_malloc, size); void* calloc(size_t size, size_t n) mi_attr_noexcept MI_FORWARD2(mi_calloc, size, n); void* realloc(void* p, size_t newsize) mi_attr_noexcept MI_FORWARD2(mi_realloc, p, newsize); - void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p); - //char* strdup(const char* s) MI_FORWARD1(mi_strdup, s); - //char* strndup(const char* s, size_t n) MI_FORWARD2(mi_strndup, s, n); + void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p); #endif #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) @@ -81,17 +83,42 @@ terms of the MIT license. A copy of the license can be found in the file #include void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_malloc,n); - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_malloc,n); - #if (__cplusplus >= 201703L) - void* operator new( std::size_t n, std::align_val_t align) noexcept(false) MI_FORWARD2(mi_malloc_aligned,n,align); - void* operator new[]( std::size_t n, std::align_val_t align) noexcept(false) MI_FORWARD2(mi_malloc_aligned,n,align); + void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } + void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } + + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n); + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n); + + #if (__cplusplus >= 201402L) + void operator delete (void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz); + void operator delete[](void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz); + #endif + + #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) + void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete (void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast(al)); }; + void operator delete[](void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast(al)); }; + + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast(al)); } #endif + #else // ------------------------------------------------------ - // With a C compiler we override the new/delete operators - // by defining the mangled C++ names of the operators (as + // With a C compiler we cannot override the new/delete operators + // as the standard requires calling into `get_new_handler` and/or + // throwing C++ exceptions (and we cannot do that from C). So, we + // hope the standard new uses `malloc` internally which will be + // redirected anyways. + // ------------------------------------------------------ + + #if 0 + // ------------------------------------------------------ + // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). // See // ------------------------------------------------------ @@ -110,6 +137,7 @@ terms of the MIT license. A copy of the license can be found in the file #else #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif + #endif #endif // __cplusplus @@ -121,57 +149,18 @@ extern "C" { // Posix & Unix functions definitions // ------------------------------------------------------ -#include - -#ifndef EINVAL -#define EINVAL 22 -#endif -#ifndef ENOMEM -#define ENOMEM 12 -#endif - void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); size_t malloc_size(void* p) MI_FORWARD1(mi_usable_size,p); size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); void cfree(void* p) MI_FORWARD0(mi_free, p); - -int posix_memalign(void** p, size_t alignment, size_t size) { - // TODO: the spec says we should return EINVAL also if alignment is not a power of 2. - // The spec also dictates we should not modify `*p` on an error. (issue#27) - // - if (alignment % sizeof(void*) != 0) return EINVAL; // no `p==NULL` check as it is declared as non-null - if ((alignment & (~alignment + 1)) != alignment) return EINVAL; // not a power of 2 - void* q = mi_malloc_aligned(size, alignment); - if (q==NULL && size != 0) return ENOMEM; - *p = q; - return 0; -} - -void* memalign(size_t alignment, size_t size) { - return mi_malloc_aligned(size, alignment); -} - -void* valloc(size_t size) { - return mi_malloc_aligned(size, _mi_os_page_size()); -} - -void* pvalloc(size_t size) { - size_t psize = _mi_os_page_size(); - if (size >= SIZE_MAX - psize) return NULL; // overflow - size_t asize = ((size + psize - 1) / psize) * psize; - return mi_malloc_aligned(asize, psize); -} - -void* aligned_alloc(size_t alignment, size_t size) { - return mi_malloc_aligned(size, alignment); -} - -void* reallocarray( void* p, size_t count, size_t size ) { // BSD - void* newp = mi_reallocn(p,count,size); - if (newp==NULL) errno = ENOMEM; - return newp; -} +// no forwarding here due to aliasing/name mangling issues +void* valloc(size_t size) { return mi_valloc(size); } +void* pvalloc(size_t size) { return mi_pvalloc(size); } +void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } +int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) @@ -181,18 +170,10 @@ void* reallocarray( void* p, size_t count, size_t size ) { // BSD void __libc_free(void* p) MI_FORWARD0(mi_free,p); void __libc_cfree(void* p) MI_FORWARD0(mi_free,p); - void* __libc_memalign(size_t alignment, size_t size) { - return memalign(alignment,size); - } - void* __libc_valloc(size_t size) { - return valloc(size); - } - void* __libc_pvalloc(size_t size) { - return pvalloc(size); - } - int __posix_memalign(void** p, size_t alignment, size_t size) { - return posix_memalign(p,alignment,size); - } + void* __libc_valloc(size_t size) { return mi_valloc(size); } + void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } + int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } #endif #ifdef __cplusplus diff --git a/src/alloc-posix.c b/src/alloc-posix.c new file mode 100644 index 00000000..b3185f15 --- /dev/null +++ b/src/alloc-posix.c @@ -0,0 +1,82 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018,2019, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// ------------------------------------------------------------------------ +// mi prefixed publi definitions of various Posix, Unix, and C++ functions +// for convenience and used when overriding these functions. +// ------------------------------------------------------------------------ + +#include "mimalloc.h" +#include "mimalloc-internal.h" + +// ------------------------------------------------------ +// Posix & Unix functions definitions +// ------------------------------------------------------ + +#include + +#ifndef EINVAL +#define EINVAL 22 +#endif +#ifndef ENOMEM +#define ENOMEM 12 +#endif + + +size_t mi_malloc_size(const void* p) mi_attr_noexcept { + return mi_usable_size(p); +} + +size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + return mi_usable_size(p); +} + +void mi_cfree(void* p) mi_attr_noexcept { + mi_free(p); +} + +int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { + // Note: The spec dictates we should not modify `*p` on an error. (issue#27) + // + if (p == NULL) return EINVAL; + if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment + if ((alignment & (alignment - 1)) != 0) return EINVAL; // not a power of 2 + void* q = mi_malloc_aligned(size, alignment); + if (q==NULL && size != 0) return ENOMEM; + *p = q; + return 0; +} + +int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { + return mi_posix_memalign(p, alignment, size); +} + +void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept { + return mi_malloc_aligned(size, alignment); +} + +void* mi_valloc(size_t size) mi_attr_noexcept { + return mi_malloc_aligned(size, _mi_os_page_size()); +} + +void* mi_pvalloc(size_t size) mi_attr_noexcept { + size_t psize = _mi_os_page_size(); + if (size >= SIZE_MAX - psize) return NULL; // overflow + size_t asize = ((size + psize - 1) / psize) * psize; + return mi_malloc_aligned(asize, psize); +} + +void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { + return mi_malloc_aligned(size, alignment); +} + +void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD + void* newp = mi_reallocn(p,count,size); + if (newp==NULL) errno = ENOMEM; + return newp; +} + diff --git a/src/alloc.c b/src/alloc.c index e8543a96..6445f473 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -106,12 +106,12 @@ void* mi_zalloc(size_t size) mi_attr_noexcept { // multi-threaded free static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) { - mi_thread_free_t tfree; - mi_thread_free_t tfreex; + mi_thread_free_t tfree = {0}; + mi_thread_free_t tfreex = {0}; bool use_delayed; do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE); if (mi_unlikely(use_delayed)) { // unlikely: this only happens on the first concurrent free in a page that is in the full list @@ -143,7 +143,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc // and reset the MI_DELAYED_FREEING flag do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; tfreex.delayed = MI_NO_DELAYED_FREE; } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value)); } @@ -177,7 +177,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block // Adjust a block that was allocated aligned, to the actual start of the block in the page. -mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, void* p) { +mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) { mi_assert_internal(page!=NULL && p!=NULL); size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL); size_t adjust = (diff % page->block_size); @@ -256,7 +256,7 @@ void _mi_free_delayed_block(mi_block_t* block) { } // Bytes available in a block -size_t mi_usable_size(void* p) mi_attr_noexcept { +size_t mi_usable_size(const void* p) mi_attr_noexcept { if (p==NULL) return 0; const mi_segment_t* segment = _mi_ptr_segment(p); const mi_page_t* page = _mi_segment_page_of(segment,p); @@ -282,8 +282,11 @@ size_t mi_usable_size(void* p) mi_attr_noexcept { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc, - (void*)&mi_malloc_small, (void*)&mi_malloc, + (void*)&mi_malloc_small, + (void*)&mi_heap_malloc, + (void*)&mi_heap_zalloc, + (void*)&mi_heap_malloc_small }; @@ -294,6 +297,24 @@ void* _mi_externs[] = { // Allocation extensions // ------------------------------------------------------ +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + UNUSED_RELEASE(size); + mi_assert(size <= mi_usable_size(p)); + mi_free(p); +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free(p); +} + extern inline void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count,size,&total)) return NULL; @@ -444,3 +465,42 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name); } + + +#ifdef __cplusplus +#include + +static mi_decl_noinline void* mi_new_try(std::size_t n) noexcept(false) { + void* p; + do { + std::new_handler h = std::get_new_handler(); + if (h==NULL) throw std::bad_alloc(); + h(); + // and try again + p = mi_malloc(n); + } while (p==NULL); + return p; +} + +// spit out `new_try` for better assembly code +void* mi_new(std::size_t n) noexcept(false) { + void* p = mi_malloc(n); + if (mi_likely(p != NULL)) return p; + else return mi_new_try(n); +} + +#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) +// for aligned allocation its fine as it is not inlined anyways +void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) { + void* p; + while ((p = mi_malloc_aligned(n,static_cast(alignment))) == NULL) { + std::new_handler h = std::get_new_handler(); + if (h==NULL) throw std::bad_alloc(); + h(); + // and try again + }; + return p; +} +#endif + +#endif diff --git a/src/heap.c b/src/heap.c index 57cb4f7d..93968713 100644 --- a/src/heap.c +++ b/src/heap.c @@ -84,7 +84,7 @@ typedef enum mi_collect_e { static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { UNUSED(arg2); UNUSED(heap); - mi_collect_t collect = (mi_collect_t)arg_collect; + mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page); if (mi_page_all_free(page)) { // no more used blocks, free the page. TODO: should we retire here and be less aggressive? @@ -131,7 +131,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) _mi_heap_delayed_free(heap); // collect all pages owned by this thread - mi_heap_visit_pages(heap, &mi_heap_page_collect, (void*)(collect), NULL); + mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != ABANDON || heap->thread_delayed_free == NULL ); // collect segment caches @@ -480,7 +480,7 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa // Visit all heap pages as areas static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) { if (visitor == NULL) return false; - return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, visitor, arg); + return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, (void*)(visitor), arg); // note: function pointer to void* :-{ } // Just to pass arguments diff --git a/src/init.c b/src/init.c index 0c77a09b..d216731a 100644 --- a/src/init.c +++ b/src/init.c @@ -426,7 +426,7 @@ static void mi_process_done(void) { // C++: use static initialization to detect process start static bool _mi_process_init(void) { mi_process_init(); - return (mi_main_thread_id != 0); + return (_mi_heap_main.thread_id != 0); } static bool mi_initialized = _mi_process_init(); diff --git a/src/page-queue.c b/src/page-queue.c index e4e14b7a..291e6edc 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -56,7 +56,7 @@ static inline uint8_t mi_bsr32(uint32_t x); #include static inline uint8_t mi_bsr32(uint32_t x) { uint32_t idx; - _BitScanReverse(&idx, x); + _BitScanReverse((DWORD*)&idx, x); return idx; } #elif defined(__GNUC__) || defined(__clang__) diff --git a/src/page.c b/src/page.c index ea86c138..04464fc5 100644 --- a/src/page.c +++ b/src/page.c @@ -114,7 +114,7 @@ void _mi_page_use_delayed_free(mi_page_t* page, bool enable) { mi_thread_free_t tfreex; do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; tfreex.delayed = (enable ? MI_USE_DELAYED_FREE : MI_NO_DELAYED_FREE); if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) { mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. @@ -137,10 +137,10 @@ void _mi_page_use_delayed_free(mi_page_t* page, bool enable) { static void mi_page_thread_free_collect(mi_page_t* page) { mi_block_t* head; - mi_thread_free_t tfree; - mi_thread_free_t tfreex; + mi_thread_free_t tfree = {0}; + mi_thread_free_t tfreex = {0}; do { - tfreex = tfree = page->thread_free; + tfreex.value = tfree.value = page->thread_free.value; head = (mi_block_t*)((uintptr_t)tfree.head << MI_TF_PTR_SHIFT); tfreex.head = 0; } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value)); diff --git a/src/static.c b/src/static.c index 4096a8af..f1656fa9 100644 --- a/src/static.c +++ b/src/static.c @@ -21,5 +21,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "heap.c" #include "alloc.c" #include "alloc-aligned.c" +#include "alloc-posix.c" #include "init.c" #include "options.c" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dd453458..a9efaff9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,17 +13,25 @@ if (NOT CMAKE_BUILD_TYPE) endif() # Import mimalloc (if installed) -find_package(mimalloc 1.0 REQUIRED) +find_package(mimalloc 1.0 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) + +message(STATUS "${MIMALLOC_INCLUDE_DIR}") # Tests -add_executable(static-override main-override.c) -target_link_libraries(static-override PUBLIC mimalloc-static) - -add_executable(static-override-cxx main-override.cpp) -target_link_libraries(static-override-cxx PUBLIC mimalloc-static) - add_executable(dynamic-override main-override.c) target_link_libraries(dynamic-override PUBLIC mimalloc) add_executable(dynamic-override-cxx main-override.cpp) target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) + +# with a static library +add_executable(static-override main-override.c) +target_link_libraries(static-override PUBLIC mimalloc-static) + +add_executable(static-override-cxx main-override.cpp) +target_link_libraries(static-override-cxx PUBLIC mimalloc-static) + +# and with a static object file; need to link with pthread explicitly :-( +add_executable(static-override-obj main-override.c $) +target_include_directories(static-override-obj PUBLIC $) +target_link_libraries(static-override-obj PUBLIC pthread) diff --git a/test/main-override.cpp b/test/main-override.cpp index c42884be..ee3d22ab 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -21,7 +21,7 @@ public: }; int main() { - mi_stats_reset(); + mi_stats_reset(); atexit(free_p); void* p1 = malloc(78); void* p2 = malloc(24); @@ -36,8 +36,11 @@ int main() { free(s); Test* t = new Test(42); delete t; + int err = mi_posix_memalign(&p1,32,60); + if (!err) free(p1); + free(p); mi_collect(true); - // mi_stats_print(NULL); // MIMALLOC_VERBOSE env is set to 2 + mi_stats_print(NULL); // MIMALLOC_VERBOSE env is set to 2 return 0; } diff --git a/test/main.cpp b/test/main.cpp deleted file mode 100644 index 4eab4edb..00000000 --- a/test/main.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include -#include - -int main() { - void* p1 = rc_malloc(16); - void* p2 = rc_malloc(16); - rc_free(p1); - rc_free(p2); - p1 = rc_malloc(16); - p2 = rc_malloc(16); - rc_free(p1); - rc_free(p2); - rc_collect(true); - rc_stats_print(); - return 0; -} diff --git a/test/test-api.c b/test/test-api.c index 643e3258..ce8d5585 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -87,33 +87,33 @@ int main() { // --------------------------------------------------- #if defined(MI_MALLOC_OVERRIDE) && !defined(_WIN32) CHECK_BODY("posix_memalign1", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, sizeof(void*), 32); - mi_assert((err==0 && (uintptr_t)p % sizeof(void*) == 0) || p==&main); + mi_assert((err==0 && (uintptr_t)p % sizeof(void*) == 0) || p==&p); mi_free(p); result = (err==0); }); CHECK_BODY("posix_memalign_no_align", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, 3, 32); - mi_assert(p==&main); + mi_assert(p==&p); result = (err==EINVAL); }); CHECK_BODY("posix_memalign_zero", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, sizeof(void*), 0); mi_free(p); result = (err==0); }); CHECK_BODY("posix_memalign_nopow2", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, 3*sizeof(void*), 32); - result = (err==EINVAL && p==&main); + result = (err==EINVAL && p==&p); }); CHECK_BODY("posix_memalign_nomem", { - void* p = &main; + void* p = &p; int err = posix_memalign(&p, sizeof(void*), SIZE_MAX); - result = (err==ENOMEM && p==&main); + result = (err==ENOMEM && p==&p); }); #endif