From fefdce3ee47ca6b833191a4364e0db696516c712 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 15 May 2024 21:12:54 +0200 Subject: [PATCH 01/34] macOS: Fix linking statically `__attribute__((constructor))` does not mark the symbol as used, so the linker ends up dead-stripping the symbol when linked statically. Adding the `used` attribute fixes that. --- src/prim/osx/alloc-override-zone.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index 9a317750..1515b886 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -422,6 +422,7 @@ __attribute__((constructor(0))) #else __attribute__((constructor)) // seems not supported by g++-11 on the M1 #endif +__attribute__((used)) static void _mi_macos_override_malloc(void) { malloc_zone_t* purgeable_zone = NULL; From 44b65b19df9107c45147df31fcafedf135619411 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 13:30:33 -0700 Subject: [PATCH 02/34] remove pre_size parameter for slices --- src/segment.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/segment.c b/src/segment.c index 9ac22f15..9e1b39a2 100644 --- a/src/segment.c +++ b/src/segment.c @@ -347,7 +347,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa } -static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, size_t* info_slices) { +static size_t mi_segment_calculate_slices(size_t required, size_t* info_slices) { size_t page_size = _mi_os_page_size(); size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size); size_t guardsize = 0; @@ -361,7 +361,6 @@ static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, siz } } - if (pre_size != NULL) *pre_size = isize; isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE); if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE; size_t segment_size = (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + guardsize, MI_SEGMENT_SLICE_SIZE) ); @@ -808,7 +807,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren ----------------------------------------------------------- */ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id, - size_t* psegment_slices, size_t* ppre_size, size_t* pinfo_slices, + size_t* psegment_slices, size_t* pinfo_slices, bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { @@ -825,7 +824,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment align_offset = _mi_align_up( info_size, MI_SEGMENT_ALIGN ); const size_t extra = align_offset - info_size; // recalculate due to potential guard pages - *psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices); + *psegment_slices = mi_segment_calculate_slices(required + extra, pinfo_slices); mi_assert_internal(*psegment_slices > 0 && *psegment_slices <= UINT32_MAX); } @@ -874,8 +873,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi // calculate needed sizes first size_t info_slices; - size_t pre_size; - size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices); + size_t segment_slices = mi_segment_calculate_slices(required, &info_slices); mi_assert_internal(segment_slices > 0 && segment_slices <= UINT32_MAX); // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) @@ -887,7 +885,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi // Allocate the segment from the OS mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id, - &segment_slices, &pre_size, &info_slices, commit, tld, os_tld); + &segment_slices, &info_slices, commit, tld, os_tld); if (segment == NULL) return NULL; // zero the segment info? -- not always needed as it may be zero initialized from the OS @@ -915,8 +913,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi if (MI_SECURE>0) { // in secure mode, we set up a protected page in between the segment info // and the page data, and at the end of the segment. - size_t os_pagesize = _mi_os_page_size(); - mi_assert_internal(mi_segment_info_size(segment) - os_pagesize >= pre_size); + size_t os_pagesize = _mi_os_page_size(); _mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize); uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize; mi_segment_ensure_committed(segment, end, os_pagesize, tld->stats); From 4a26a4568e0f593b7842d91fbf4ec5f80d06bc65 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 14:26:05 -0700 Subject: [PATCH 03/34] fix out-of-bounds write on span free in huge segments --- src/segment.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index 9e1b39a2..6044c270 100644 --- a/src/segment.c +++ b/src/segment.c @@ -623,7 +623,9 @@ static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size mi_assert_internal(slice->slice_count == slice_count); // no overflow? slice->slice_offset = 0; if (slice_count > 1) { - mi_slice_t* last = &segment->slices[slice_index + slice_count - 1]; + mi_slice_t* last = slice + slice_count - 1; + mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment); + if (last > end) { last = end; } last->slice_count = 0; last->slice_offset = (uint32_t)(sizeof(mi_page_t)*(slice_count - 1)); last->block_size = 0; From a38c8dd0f9fb39057f8cac3a30f265fea1ec4db1 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 14:41:10 -0700 Subject: [PATCH 04/34] rename must_free_whole -> has_partial_free --- include/mimalloc/prim.h | 2 +- src/os.c | 12 +++++------- src/prim/emscripten/prim.c | 2 +- src/prim/unix/prim.c | 2 +- src/prim/wasi/prim.c | 2 +- src/prim/windows/prim.c | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 4d813b7f..9561335a 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -26,7 +26,7 @@ typedef struct mi_os_mem_config_s { size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows) size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) bool has_overcommit; // can we reserve more memory than can be actually committed? - bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc) + bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc) bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) } mi_os_mem_config_t; diff --git a/src/os.c b/src/os.c index 09ae367d..88e7fcb3 100644 --- a/src/os.c +++ b/src/os.c @@ -11,9 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- - Initialization. - On windows initializes support for aligned allocation and - large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). + Initialization. ----------------------------------------------------------- */ static mi_os_mem_config_t mi_os_mem_config = { @@ -21,7 +19,7 @@ static mi_os_mem_config_t mi_os_mem_config = { 0, // large page size (usually 2MiB) 4096, // allocation granularity true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) - false, // must free whole? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) + false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) }; @@ -254,7 +252,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; - if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block + if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block // over-allocate uncommitted (virtual) memory p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats); if (p == NULL) return NULL; @@ -275,7 +273,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats); if (p == NULL) return NULL; - // and selectively unmap parts around the over-allocated area. (noop on sbrk) + // and selectively unmap parts around the over-allocated area. void* aligned_p = mi_align_up_ptr(p, alignment); size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; size_t mid_size = _mi_align_up(size, _mi_os_page_size()); @@ -283,7 +281,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); } if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); } - // we can return the aligned pointer on `mmap` (and sbrk) systems + // we can return the aligned pointer on `mmap` systems p = aligned_p; *base = aligned_p; // since we freed the pre part, `*base == p`. } diff --git a/src/prim/emscripten/prim.c b/src/prim/emscripten/prim.c index 1f60a1bb..f3797c9e 100644 --- a/src/prim/emscripten/prim.c +++ b/src/prim/emscripten/prim.c @@ -51,7 +51,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) { config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB config->alloc_granularity = 16; config->has_overcommit = false; - config->must_free_whole = true; + config->has_partial_free = false; config->has_virtual_reserve = false; } diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index a7812cb6..8093fbdd 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -144,7 +144,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) } config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this? config->has_overcommit = unix_detect_overcommit(); - config->must_free_whole = false; // mmap can free in parts + config->has_partial_free = true; // mmap can free in parts config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE) // disable transparent huge pages for this process? diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index f74acd2a..e95f67f5 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -23,7 +23,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) { config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB config->alloc_granularity = 16; config->has_overcommit = false; - config->must_free_whole = true; + config->has_partial_free = false; config->has_virtual_reserve = false; } diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 2dd7c602..05aab43a 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -112,7 +112,7 @@ static bool win_enable_large_os_pages(size_t* large_page_size) void _mi_prim_mem_init( mi_os_mem_config_t* config ) { config->has_overcommit = false; - config->must_free_whole = true; + config->has_partial_free = false; config->has_virtual_reserve = true; // get the page size SYSTEM_INFO si; From 6e960f06e760c1d47e26904cca83476ee3953ec2 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 15:13:03 -0700 Subject: [PATCH 05/34] add note on using linker section instead of fiber api --- src/prim/windows/prim.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 05aab43a..e82ad969 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -572,6 +572,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #if !defined(MI_SHARED_LIB) // use thread local storage keys to detect thread ending +// note: another design could be to use special linker sections (see issue #869) #include #if (_WIN32_WINNT < 0x600) // before Windows Vista WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); From 5eb29e971b0238d7a313f4129ba76589ea10f4c8 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 18:52:29 -0700 Subject: [PATCH 06/34] bump version to 1.8.7 for further development --- cmake/mimalloc-config-version.cmake | 2 +- include/mimalloc.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 955336a4..f7255251 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 1) set(mi_version_minor 8) -set(mi_version_patch 6) +set(mi_version_patch 7) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index 3ab69268..af885e2e 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 186 // major + 2 digits minor +#define MI_MALLOC_VERSION 187 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes From e7f2ffe26cc7211d8be603684a9cbd6ee5e5dd3e Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 18:53:50 -0700 Subject: [PATCH 07/34] fix build warning on alpine 32-bit --- src/init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/init.c b/src/init.c index f5fa2ad8..5bdadabc 100644 --- a/src/init.c +++ b/src/init.c @@ -33,6 +33,9 @@ const mi_page_t _mi_page_empty = { MI_ATOMIC_VAR_INIT(0), // xthread_free MI_ATOMIC_VAR_INIT(0), // xheap NULL, NULL + #if MI_INTPTR_SIZE==4 + , { NULL } + #endif }; #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) From e3fea8f4e1a04f4b260a78e6b5a8f0eead990445 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 18:58:18 -0700 Subject: [PATCH 08/34] temporary fix for musl libc 32-bit compilation on alpine (see issue #895) --- src/arena.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index 511fe2fd..ed6a6a27 100644 --- a/src/arena.c +++ b/src/arena.c @@ -174,7 +174,10 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st *memid = _mi_memid_none(); // try static - void* p = mi_arena_static_zalloc(size, MI_MAX_ALIGN_SIZE, memid); + void* p = NULL; + #if !(MI_INTPTR_SIZE==4 && MI_LIBC_MUSL) // fix 32-bit musl compilation, issue #895 + p = mi_arena_static_zalloc(size, MI_MAX_ALIGN_SIZE, memid); + #endif if (p != NULL) return p; // or fall back to the OS From aa91fe0f1c5bba277c874a24ac2301f0891255b6 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 19:05:24 -0700 Subject: [PATCH 09/34] fix build warning on 32-bit musl --- src/arena.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index ed6a6a27..ce23b89a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -175,7 +175,9 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st // try static void* p = NULL; - #if !(MI_INTPTR_SIZE==4 && MI_LIBC_MUSL) // fix 32-bit musl compilation, issue #895 + #if (MI_INTPTR_SIZE==4 && MI_LIBC_MUSL) // fix 32-bit musl compilation, issue #895 + MI_UNUSED(mi_arena_static_zalloc); + #else p = mi_arena_static_zalloc(size, MI_MAX_ALIGN_SIZE, memid); #endif if (p != NULL) return p; From 97013ee2b493c8bcf29ee9d71dbd8ce74fcef54b Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 16 May 2024 19:16:44 -0700 Subject: [PATCH 10/34] improve static allocation --- src/arena.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index ce23b89a..acfde132 100644 --- a/src/arena.c +++ b/src/arena.c @@ -147,12 +147,13 @@ static _Atomic(size_t) mi_arena_static_top; static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) { *memid = _mi_memid_none(); if (size == 0 || size > MI_ARENA_STATIC_MAX) return NULL; - if ((mi_atomic_load_relaxed(&mi_arena_static_top) + size) > MI_ARENA_STATIC_MAX) return NULL; + const size_t toplow = mi_atomic_load_relaxed(&mi_arena_static_top); + if ((toplow + size) > MI_ARENA_STATIC_MAX) return NULL; // try to claim space if (alignment == 0) { alignment = 1; } const size_t oversize = size + alignment - 1; - if (oversize > MI_ARENA_STATIC_MAX) return NULL; + if (toplow + oversize > MI_ARENA_STATIC_MAX) return NULL; const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize); size_t top = oldtop + oversize; if (top > MI_ARENA_STATIC_MAX) { From 8fd1184272fdf5c93f4776a36086911b55fb315e Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 17 May 2024 09:06:27 -0700 Subject: [PATCH 11/34] fix alignment issue on Alpine 32-bit musl compilation (issue #895) --- src/arena.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/arena.c b/src/arena.c index acfde132..aa4bfa42 100644 --- a/src/arena.c +++ b/src/arena.c @@ -141,8 +141,8 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit #define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit -static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; -static _Atomic(size_t) mi_arena_static_top; +static mi_decl_cache_align uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; // must be cache aligned, see issue #895 +static mi_decl_cache_align _Atomic(size_t) mi_arena_static_top; static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) { *memid = _mi_memid_none(); @@ -151,7 +151,7 @@ static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* m if ((toplow + size) > MI_ARENA_STATIC_MAX) return NULL; // try to claim space - if (alignment == 0) { alignment = 1; } + if (alignment < MI_MAX_ALIGN_SIZE) { alignment = MI_MAX_ALIGN_SIZE; } const size_t oversize = size + alignment - 1; if (toplow + oversize > MI_ARENA_STATIC_MAX) return NULL; const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize); @@ -167,7 +167,7 @@ static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* m memid->initially_zero = true; const size_t start = _mi_align_up(oldtop, alignment); uint8_t* const p = &mi_arena_static[start]; - _mi_memzero(p, size); + _mi_memzero_aligned(p, size); return p; } @@ -175,12 +175,7 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st *memid = _mi_memid_none(); // try static - void* p = NULL; - #if (MI_INTPTR_SIZE==4 && MI_LIBC_MUSL) // fix 32-bit musl compilation, issue #895 - MI_UNUSED(mi_arena_static_zalloc); - #else - p = mi_arena_static_zalloc(size, MI_MAX_ALIGN_SIZE, memid); - #endif + void* p = mi_arena_static_zalloc(size, MI_MAX_ALIGN_SIZE, memid); if (p != NULL) return p; // or fall back to the OS From a05e1f4868dc2e64deacd2119e707f7ae90a17c6 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 17 May 2024 09:12:03 -0700 Subject: [PATCH 12/34] comment to not add custom fields (issue #896) --- src/arena.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index aa4bfa42..25ce56ec 100644 --- a/src/arena.c +++ b/src/arena.c @@ -51,12 +51,13 @@ typedef struct mi_arena_s { bool exclusive; // only allow allocations if specifically for this arena bool is_large; // memory area consists of large- or huge OS pages (always committed) _Atomic(size_t) search_idx; // optimization to start the search for free blocks - _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. + _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) + // do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields. } mi_arena_t; From e58fa376d42c632347c68b256cce1607f859dfa1 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 17 May 2024 13:19:37 -0700 Subject: [PATCH 13/34] on windows, retry virtual alloc if out-of-memory, see issue #894 --- src/prim/windows/prim.c | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index e82ad969..8f9dbd19 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -178,7 +178,7 @@ int _mi_prim_free(void* addr, size_t size ) { // VirtualAlloc //--------------------------------------------- -static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { +static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { @@ -200,13 +200,48 @@ static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignmen param.Arg.Pointer = &reqs; void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); if (p != NULL) return p; - _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); + _mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags); // fall through on error } // last resort return VirtualAlloc(addr, size, flags, PAGE_READWRITE); } +static bool win_is_out_of_memory_error(DWORD err) { + switch (err) { + case ERROR_COMMITMENT_MINIMUM: + case ERROR_COMMITMENT_LIMIT: + case ERROR_PAGEFILE_QUOTA: + case ERROR_NOT_ENOUGH_MEMORY: + return true; + default: + return false; + } +} + +static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { + for (DWORD tries = 1; tries <= 5; tries++) { + void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags); + if (p != NULL) { + // success, return the address + return p; + } + else if (try_alignment < 2*MI_SEGMENT_ALIGN && + (flags&MEM_COMMIT)!=0 && (flags&MEM_LARGE_PAGES)==0 && + win_is_out_of_memory_error(GetLastError())) { + // if committing regular memory and being out-of-memory, + // keep trying for a bit in case memory frees up after all. See issue #894 + _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags); + Sleep(tries*25 /* milliseconds */); // try for at most (1+2+3+4+5)x25 = 375ms + } + else { + // otherwise return with an error + break; + } + } + return NULL; +} + static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { mi_assert_internal(!(large_only && !allow_large)); static _Atomic(size_t) large_page_try_ok; // = 0; From 1b21415dfa6a5710202bb7ab2c37fe653f180ba0 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 09:56:02 -0700 Subject: [PATCH 14/34] make retry on oom an option; revise size options to not overflow the long --- include/mimalloc.h | 5 +++-- include/mimalloc/internal.h | 4 ++-- src/options.c | 41 +++++++++++++++++++++++-------------- src/prim/windows/prim.c | 19 ++++++++++------- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index af885e2e..ae6f99b4 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -327,7 +327,7 @@ typedef enum mi_option_e { mi_option_allow_large_os_pages, // allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process. mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node - mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup + mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`) mi_option_deprecated_segment_cache, mi_option_deprecated_page_reset, mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination @@ -341,11 +341,12 @@ typedef enum mi_option_e { mi_option_max_warnings, // issue at most N warning messages mi_option_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%) mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe - mi_option_arena_reserve, // initial memory size in KiB for arena reservation (= 1 GiB on 64-bit) + mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`) mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10) mi_option_purge_extend_delay, mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1) mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) + mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 688dba0b..066479ac 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -358,10 +358,10 @@ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { } #else /* __builtin_umul_overflow is unavailable */ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) + #define MI_MUL_COULD_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) *total = count * size; // note: gcc/clang optimize this to directly check the overflow flag - return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); + return ((size >= MI_MUL_COULD_OVERFLOW || count >= MI_MUL_COULD_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); } #endif diff --git a/src/options.c b/src/options.c index 78e9377c..db6e040f 100644 --- a/src/options.c +++ b/src/options.c @@ -65,7 +65,7 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve OS memory in advance + { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION(abandoned_page_purge) }, // purge free page memory when a thread terminates @@ -79,20 +79,21 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output - { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output - { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments per try. + { 32, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output + { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! #if (MI_INTPTR_SIZE>4) - { 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time + { 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) #else - { 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, + { 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit #endif { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free { 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. }; static void mi_option_init(mi_option_desc_t* desc); @@ -136,8 +137,12 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { mi_assert_internal(mi_option_has_size_in_kib(option)); - long x = mi_option_get(option); - return (x < 0 ? 0 : (size_t)x * MI_KiB); + const long x = mi_option_get(option); + size_t size = (x < 0 ? 0 : (size_t)x); + if (mi_option_has_size_in_kib(option)) { + size *= MI_KiB; + } + return size; } void mi_option_set(mi_option_t option, long value) { @@ -480,14 +485,20 @@ static void mi_option_init(mi_option_desc_t* desc) { else { char* end = buf; long value = strtol(buf, &end, 10); - if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_arena_reserve) { - // this option is interpreted in KiB to prevent overflow of `long` + if (mi_option_has_size_in_kib(desc->option)) { + // this option is interpreted in KiB to prevent overflow of `long` for large allocations + // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) + size_t size = (value < 0 ? 0 : (size_t)value); + bool overflow = false; if (*end == 'K') { end++; } - else if (*end == 'M') { value *= MI_KiB; end++; } - else if (*end == 'G') { value *= MI_MiB; end++; } - else { value = (value + MI_KiB - 1) / MI_KiB; } - if (end[0] == 'I' && end[1] == 'B') { end += 2; } - else if (*end == 'B') { end++; } + else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; } + else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; } + else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; } + else { size = (size + MI_KiB - 1) / MI_KiB; } + if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB + else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb + if (overflow || size > MI_MAX_ALLOC_SIZE) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); } + value = (size > LONG_MAX ? LONG_MAX : (long)size); } if (*end == 0) { desc->value = value; diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 8f9dbd19..5074ad4c 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -220,19 +220,24 @@ static bool win_is_out_of_memory_error(DWORD err) { } static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { - for (DWORD tries = 1; tries <= 5; tries++) { + long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds + if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to "true" + for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms) void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags); - if (p != NULL) { + if (p != NULL) { // success, return the address - return p; + return p; } - else if (try_alignment < 2*MI_SEGMENT_ALIGN && - (flags&MEM_COMMIT)!=0 && (flags&MEM_LARGE_PAGES)==0 && - win_is_out_of_memory_error(GetLastError())) { + else if (max_retry_msecs > 0 && (try_alignment <= 2*MI_SEGMENT_ALIGN) && + (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 && + win_is_out_of_memory_error(GetLastError())) { // if committing regular memory and being out-of-memory, // keep trying for a bit in case memory frees up after all. See issue #894 _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags); - Sleep(tries*25 /* milliseconds */); // try for at most (1+2+3+4+5)x25 = 375ms + long sleep_msecs = tries*40; // increasing waits + if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; } + max_retry_msecs -= sleep_msecs; + Sleep(sleep_msecs); } else { // otherwise return with an error From d9441ffce237ff54d74579ac6622fe4ff7e65e38 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:16:36 -0700 Subject: [PATCH 15/34] fix compilation on AIX, upstream of python/cpython#111593 --- src/prim/unix/prim.c | 61 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 8093fbdd..7e20c6a9 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -28,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // mmap #include // sysconf #include // open, close, read, access - + #if defined(__linux__) #include #if defined(MI_NO_THP) @@ -57,7 +57,8 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(__OpenBSD__) && !defined(__sun) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && \ + !defined(__OpenBSD__) && !defined(__sun) && !defined(_AIX) #define MI_HAS_SYSCALL_H #include #endif @@ -65,7 +66,7 @@ terms of the MIT license. A copy of the license can be found in the file //------------------------------------------------------------------------------------ // Use syscalls for some primitives to allow for libraries that override open/read/close etc. -// and do allocation themselves; using syscalls prevents recursion when mimalloc is +// and do allocation themselves; using syscalls prevents recursion when mimalloc is // still initializing (issue #713) //------------------------------------------------------------------------------------ @@ -85,8 +86,8 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__sun) && \ - (!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS and Solaris +#elif !defined(__sun) && !defined(_AIX) && \ + (!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS et al. static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); @@ -130,12 +131,12 @@ static bool unix_detect_overcommit(void) { os_overcommit = (val != 0); } #else - // default: overcommit is true + // default: overcommit is true #endif return os_overcommit; } -void _mi_prim_mem_init( mi_os_mem_config_t* config ) +void _mi_prim_mem_init( mi_os_mem_config_t* config ) { long psize = sysconf(_SC_PAGESIZE); if (psize > 0) { @@ -197,12 +198,12 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); - if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris @@ -218,7 +219,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p void* hint = _mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { p = mmap(hint, size, protect_flags, flags, fd, 0); - if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { + if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly? int err = 0; #else @@ -227,7 +228,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); } if (p!=MAP_FAILED) return p; - // fall back to regular mmap + // fall back to regular mmap } } #endif @@ -356,9 +357,9 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); - + *is_zero = true; - int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); + int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); return (*addr != NULL ? 0 : errno); } @@ -381,8 +382,8 @@ static void unix_mprotect_hint(int err) { } - - + + int _mi_prim_commit(void* start, size_t size, bool* is_zero) { // commit: ensure we can access the area @@ -390,19 +391,19 @@ int _mi_prim_commit(void* start, size_t size, bool* is_zero) { // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but // we sometimes call commit on a range with still partially committed // memory and `mprotect` does not zero the range. - *is_zero = false; + *is_zero = false; int err = mprotect(start, size, (PROT_READ | PROT_WRITE)); - if (err != 0) { - err = errno; + if (err != 0) { + err = errno; unix_mprotect_hint(err); } return err; } int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { - int err = 0; + int err = 0; // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) - err = unix_madvise(start, size, MADV_DONTNEED); + err = unix_madvise(start, size, MADV_DONTNEED); #if !MI_DEBUG && !MI_SECURE *needs_recommit = false; #else @@ -414,15 +415,15 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { *needs_recommit = true; const int fd = unix_mmap_fd(); void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); - if (p != start) { err = errno; } + if (p != start) { err = errno; } */ return err; } int _mi_prim_reset(void* start, size_t size) { - // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it + // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it // will not reduce the `rss` stats in tools like `top` even though the memory is available - // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by + // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by // default `MADV_DONTNEED` is used though. #if defined(MADV_FREE) static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); @@ -442,7 +443,7 @@ int _mi_prim_reset(void* start, size_t size) { int _mi_prim_protect(void* start, size_t size, bool protect) { int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); - if (err != 0) { err = errno; } + if (err != 0) { err = errno; } unix_mprotect_hint(err); return err; } @@ -483,7 +484,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo if (err != 0) { err = errno; _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); - } + } } return (*addr != NULL ? 0 : errno); } @@ -598,9 +599,9 @@ mi_msecs_t _mi_prim_clock_now(void) { // low resolution timer mi_msecs_t _mi_prim_clock_now(void) { #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) - return (mi_msecs_t)clock(); + return (mi_msecs_t)clock(); #elif (CLOCKS_PER_SEC < 1000) - return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); #else return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); #endif @@ -640,7 +641,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) pinfo->stime = timeval_secs(&rusage.ru_stime); #if !defined(__HAIKU__) pinfo->page_faults = rusage.ru_majflt; -#endif +#endif #if defined(__HAIKU__) // Haiku does not have (yet?) a way to // get these stats per process @@ -767,7 +768,7 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { bool _mi_prim_random_buf(void* buf, size_t buf_len) { // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf // may fail silently on macOS. See PR #390, and - return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); } #elif defined(__ANDROID__) || defined(__DragonFly__) || \ @@ -866,7 +867,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { } } -#else +#else void _mi_prim_thread_init_auto_done(void) { // nothing From 937fd19338436ad426d8abf1c39c71e57295c5fa Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:32:10 -0700 Subject: [PATCH 16/34] comment --- include/mimalloc/atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 807c4da8..afd8d647 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -132,7 +132,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { #elif defined(_MSC_VER) -// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. +// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif From 87c4012f13d06bc92869edc0a79c2eff7343aa6d Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:37:40 -0700 Subject: [PATCH 17/34] make syscall test in primitives positive and avoid unused function warnings. upstream python/cpython#111907, python/cpython#111593, python/cpython#117548 --- src/prim/unix/prim.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 7e20c6a9..1efc84dc 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -57,8 +57,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && \ - !defined(__OpenBSD__) && !defined(__sun) && !defined(_AIX) +#if defined(__linux__) #define MI_HAS_SYSCALL_H #include #endif @@ -68,37 +67,36 @@ terms of the MIT license. A copy of the license can be found in the file // Use syscalls for some primitives to allow for libraries that override open/read/close etc. // and do allocation themselves; using syscalls prevents recursion when mimalloc is // still initializing (issue #713) +// Declare inline to avoid unused function warnings. //------------------------------------------------------------------------------------ - #if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) -static int mi_prim_open(const char* fpath, int open_flags) { +static inline int mi_prim_open(const char* fpath, int open_flags) { return syscall(SYS_open,fpath,open_flags,0); } -static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { +static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { return syscall(SYS_read,fd,buf,bufsize); } -static int mi_prim_close(int fd) { +static inline int mi_prim_close(int fd) { return syscall(SYS_close,fd); } -static int mi_prim_access(const char *fpath, int mode) { +static inline int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__sun) && !defined(_AIX) && \ - (!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS et al. +#else -static int mi_prim_open(const char* fpath, int open_flags) { +static inline int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); } -static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { +static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { return read(fd,buf,bufsize); } -static int mi_prim_close(int fd) { +static inline int mi_prim_close(int fd) { return close(fd); } -static int mi_prim_access(const char *fpath, int mode) { +static inline int mi_prim_access(const char *fpath, int mode) { return access(fpath,mode); } From 999b31fea6cf144ad41d9134a1bdf3d9f1674175 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:39:04 -0700 Subject: [PATCH 18/34] allow syscall include on FreeBSD as well --- src/prim/unix/prim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 1efc84dc..a4e012c7 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -57,7 +57,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if defined(__linux__) +#if defined(__linux__) || defined(__FreeBSD__) #define MI_HAS_SYSCALL_H #include #endif From c48a21215b1a3f447b71bb29b748d5a57b462d99 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:41:39 -0700 Subject: [PATCH 19/34] fix arm64 windows compilation, upstream of python/cpython#111527 --- include/mimalloc/atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index afd8d647..d5333dd9 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -201,7 +201,7 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_ #else uintptr_t x = *p; if (mo > mi_memory_order_relaxed) { - while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; + while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; } return x; #endif From 92a8268fa42fc058a4ecbe0abe4bbd8a22b42088 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:45:23 -0700 Subject: [PATCH 20/34] make warning an trace message if we cannot allocate at a hinted address, upstream of python/cpython#113372 --- src/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index a4e012c7..99325d03 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -198,7 +198,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { int err = errno; - _mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); + _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); } if (p!=MAP_FAILED) return p; // fall back to regular mmap @@ -223,7 +223,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p #else int err = errno; #endif - _mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); + _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); } if (p!=MAP_FAILED) return p; // fall back to regular mmap From 381da05d8a69fb2b836fd3dabbd987b7385d7f6f Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 18 May 2024 16:48:58 -0700 Subject: [PATCH 21/34] make mimalloc includes relative to the current file, upstream of python/cpython#113141 --- include/mimalloc/internal.h | 4 ++-- include/mimalloc/types.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 066479ac..f184e432 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file // functions and macros. // -------------------------------------------------------------------------- -#include "mimalloc/types.h" -#include "mimalloc/track.h" +#include "types.h" +#include "track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 35a3965e..e99742ac 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -23,7 +23,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc/atomic.h" // _Atomic +#include "atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int @@ -387,7 +387,7 @@ typedef struct mi_memid_s { // Segments contain mimalloc pages // --------------------------------------------------------------- -// Segments are large allocated memory blocks (2MiB on 64 bit) from the OS. +// Segments are large allocated memory blocks (2MiB on 64 bit) from the OS. // Inside segments we allocated fixed size _pages_ that contain blocks. typedef struct mi_segment_s { // constant fields From 66052f135fa3454d3474b217ab191d6395d873da Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 18 May 2024 17:05:13 -0700 Subject: [PATCH 22/34] allow custom thread id, upstream of python/cpython#115488) --- include/mimalloc/prim.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 9561335a..126ea7cd 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -222,7 +222,13 @@ extern bool _mi_process_is_initialized; // has mi_process_init been static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; // Get a unique id for the current thread. -#if defined(_WIN32) +#if defined(MI_PRIM_THREAD_ID) + +static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { + return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) +} + +#elif defined(_WIN32) #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN From 710d6138c7c1e31ef3d6871dd42390e85ddc5c5a Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 18 May 2024 17:42:47 -0700 Subject: [PATCH 23/34] refactor thread meta-data initilazation, upstream of python/cpython#113263 --- include/mimalloc/internal.h | 2 ++ src/heap.c | 35 ++++++++++++++++++++++++----------- src/init.c | 34 +++++++++++++++++----------------- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index f184e432..2161142c 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -88,6 +88,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap void _mi_thread_done(mi_heap_t* heap); void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); // os.c void _mi_os_init(void); // called from process init @@ -183,6 +184,7 @@ size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); diff --git a/src/heap.c b/src/heap.c index 772a6051..8817887d 100644 --- a/src/heap.c +++ b/src/heap.c @@ -123,6 +123,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) const bool force = (collect >= MI_FORCE); _mi_deferred_free(heap, force); + // python/cpython#112532: we may be called from a thread that is not the owner of the heap + const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id()); + // note: never reclaim on collect but leave it to threads that need storage to reclaim if ( #ifdef NDEBUG @@ -130,7 +133,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) #else collect >= MI_FORCE #endif - && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim) + && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim) { // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. // if all memory is freed by now, all segments should be freed. @@ -157,7 +160,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) _mi_segments_collect(collect == MI_FORCE, &heap->tld->segments); // if forced, collect thread data cache on program-exit (or shared library unload) - if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { + if (force && is_main_thread && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache } @@ -201,19 +204,29 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id) { + _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); + heap->tld = tld; + heap->thread_id = _mi_thread_id(); + heap->arena_id = arena_id; + if (heap == tld->heap_backing) { + _mi_random_init(&heap->random); + } + else { + _mi_random_split(&tld->heap_backing->random, &heap->random); + } + heap->cookie = _mi_heap_random_next(heap) | 1; + heap->keys[0] = _mi_heap_random_next(heap); + heap->keys[1] = _mi_heap_random_next(heap); +} + mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap == NULL) return NULL; - _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); - heap->tld = bheap->tld; - heap->thread_id = _mi_thread_id(); - heap->arena_id = arena_id; - _mi_random_split(&bheap->random, &heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; - heap->keys[0] = _mi_heap_random_next(heap); - heap->keys[1] = _mi_heap_random_next(heap); - heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe + _mi_heap_init(heap, bheap->tld, arena_id); + // don't reclaim abandoned pages or otherwise destroy is unsafe + heap->no_reclaim = true; // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; diff --git a/src/init.c b/src/init.c index 5bdadabc..9d8632e4 100644 --- a/src/init.c +++ b/src/init.c @@ -264,7 +264,7 @@ void _mi_thread_data_collect(void) { } // Initialize the thread local default heap, called from `mi_thread_init` -static bool _mi_heap_init(void) { +static bool _mi_thread_heap_init(void) { if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; if (_mi_is_main_thread()) { // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization @@ -280,25 +280,25 @@ static bool _mi_heap_init(void) { mi_tld_t* tld = &td->tld; mi_heap_t* heap = &td->heap; - _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); - heap->thread_id = _mi_thread_id(); - _mi_random_init(&heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; - heap->keys[0] = _mi_heap_random_next(heap); - heap->keys[1] = _mi_heap_random_next(heap); - heap->tld = tld; - tld->heap_backing = heap; - tld->heaps = heap; - tld->segments.stats = &tld->stats; - tld->segments.os = &tld->os; - tld->os.stats = &tld->stats; - _mi_heap_set_default_direct(heap); + _mi_tld_init(tld, heap); // must be before `_mi_heap_init` + _mi_heap_init(heap, tld, _mi_arena_id_none()); + _mi_heap_set_default_direct(heap); } return false; } +// initialize thread local data +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { + _mi_memzero_aligned(tld,sizeof(mi_tld_t)); + tld->heap_backing = bheap; + tld->heaps = bheap; + tld->segments.stats = &tld->stats; + tld->segments.os = &tld->os; + tld->os.stats = &tld->stats; +} + // Free the thread local default heap (called from `mi_thread_done`) -static bool _mi_heap_done(mi_heap_t* heap) { +static bool _mi_thread_heap_done(mi_heap_t* heap) { if (!mi_heap_is_initialized(heap)) return true; // reset default heap @@ -392,7 +392,7 @@ void mi_thread_init(void) mi_attr_noexcept // initialize the thread local default heap // (this will call `_mi_heap_set_default_direct` and thus set the // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) - if (_mi_heap_init()) return; // returns true if already initialized + if (_mi_thread_heap_init()) return; // returns true if already initialized _mi_stat_increase(&_mi_stats_main.threads, 1); mi_atomic_increment_relaxed(&thread_count); @@ -424,7 +424,7 @@ void _mi_thread_done(mi_heap_t* heap) if (heap->thread_id != _mi_thread_id()) return; // abandon the thread local heap - if (_mi_heap_done(heap)) return; // returns true if already ran + if (_mi_thread_heap_done(heap)) return; // returns true if already ran } void _mi_heap_set_default_direct(mi_heap_t* heap) { From 0c4041fa53e9d046558d7f5a78b156f87efc5801 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 19 May 2024 08:06:32 -0700 Subject: [PATCH 24/34] add support for custom heap and page tags, upstream of python/cpython#113742 --- include/mimalloc/internal.h | 4 +++- include/mimalloc/types.h | 2 ++ src/heap.c | 31 ++++++++++++++++++++++--------- src/init.c | 9 ++++++--- src/segment.c | 13 ++++++++++--- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 2161142c..2954eabd 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -184,12 +184,13 @@ size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" -void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id); +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); void _mi_heap_unsafe_destroy_all(void); +mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag); // "stats.c" void _mi_stats_done(mi_stats_t* stats); @@ -508,6 +509,7 @@ static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) { mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING); mi_atomic_store_release(&page->xheap,(uintptr_t)heap); + if (heap != NULL) { page->heap_tag = heap->tag; } } // Thread free flag helpers diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index e99742ac..ed326c69 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -307,6 +307,7 @@ typedef struct mi_page_s { mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) uint16_t used; // number of blocks in use (including blocks in `thread_free`) uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`) + uint8_t heap_tag; // tag of the owning heap, used for separated heaps by object type // padding size_t block_size; // size available in each block (always `>0`) uint8_t* page_start; // start of the page area containing the blocks @@ -481,6 +482,7 @@ struct mi_heap_s { size_t page_retired_max; // largest retired index into the `pages` array. mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages + uint8_t tag; // custom tag, can be used for separating heaps based on the object types mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") }; diff --git a/src/heap.c b/src/heap.c index 8817887d..f6f23549 100644 --- a/src/heap.c +++ b/src/heap.c @@ -204,11 +204,13 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } -void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id) { +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag) { _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); heap->tld = tld; - heap->thread_id = _mi_thread_id(); - heap->arena_id = arena_id; + heap->thread_id = _mi_thread_id(); + heap->arena_id = arena_id; + heap->no_reclaim = noreclaim; + heap->tag = tag; if (heap == tld->heap_backing) { _mi_random_init(&heap->random); } @@ -218,18 +220,17 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id) { heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); + // push on the thread local heaps list + heap->next = heap->tld->heaps; + heap->tld->heaps = heap; } mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap == NULL) return NULL; - _mi_heap_init(heap, bheap->tld, arena_id); - // don't reclaim abandoned pages or otherwise destroy is unsafe - heap->no_reclaim = true; - // push on the thread local heaps list - heap->next = heap->tld->heaps; - heap->tld->heaps = heap; + // don't reclaim abandoned pages or otherwise destroy is unsafe + _mi_heap_init(heap, bheap->tld, arena_id, true /* no reclaim */, 0 /* default tag */); return heap; } @@ -287,6 +288,18 @@ static void mi_heap_free(mi_heap_t* heap) { mi_free(heap); } +// return a heap on the same thread as `heap` specialized for the specified tag (if it exists) +mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { + if (heap->tag == tag) { + return heap; + } + for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) { + if (curr->tag == tag) { + return curr; + } + } + return NULL; +} /* ----------------------------------------------------------- Heap destroy diff --git a/src/init.c b/src/init.c index 9d8632e4..62bb69dd 100644 --- a/src/init.c +++ b/src/init.c @@ -25,6 +25,7 @@ const mi_page_t _mi_page_empty = { NULL, // local_free 0, // used 0, // block size shift + 0, // heap tag 0, // block_size NULL, // page_start #if (MI_PADDING || MI_ENCODE_FREELIST) @@ -108,7 +109,8 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { 0, // page count MI_BIN_FULL, 0, // page retired min/max NULL, // next - false, + false, // can reclaim + 0, // tag MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -146,6 +148,7 @@ mi_heap_t _mi_heap_main = { MI_BIN_FULL, 0, // page retired min/max NULL, // next heap false, // can reclaim + 0, // tag MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -281,7 +284,7 @@ static bool _mi_thread_heap_init(void) { mi_tld_t* tld = &td->tld; mi_heap_t* heap = &td->heap; _mi_tld_init(tld, heap); // must be before `_mi_heap_init` - _mi_heap_init(heap, tld, _mi_arena_id_none()); + _mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */); _mi_heap_set_default_direct(heap); } return false; @@ -291,7 +294,7 @@ static bool _mi_thread_heap_init(void) { void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { _mi_memzero_aligned(tld,sizeof(mi_tld_t)); tld->heap_backing = bheap; - tld->heaps = bheap; + tld->heaps = NULL; tld->segments.stats = &tld->stats; tld->segments.os = &tld->os; tld->os.stats = &tld->stats; diff --git a/src/segment.c b/src/segment.c index cfd6c1a3..fc13d2e7 100644 --- a/src/segment.c +++ b/src/segment.c @@ -714,6 +714,7 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_seg // zero the page data, but not the segment fields and capacity, page start, and block_size (for page size calculations) size_t block_size = page->block_size; uint8_t block_size_shift = page->block_size_shift; + uint8_t heap_tag = page->heap_tag; uint8_t* page_start = page->page_start; uint16_t capacity = page->capacity; uint16_t reserved = page->reserved; @@ -723,6 +724,7 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_seg page->reserved = reserved; page->block_size = block_size; page->block_size_shift = block_size_shift; + page->heap_tag = heap_tag; page->page_start = page_start; segment->used--; @@ -898,7 +900,12 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, mi_assert(page->next == NULL); _mi_stat_decrease(&tld->stats->pages_abandoned, 1); // set the heap again and allow heap thread delayed free again. - mi_page_set_heap(page, heap); + mi_heap_t* target_heap = _mi_heap_by_tag(heap, page->heap_tag); // allow custom heaps to separate objects + if (target_heap == NULL) { + target_heap = heap; + _mi_error_message(EINVAL, "page with tag %u cannot be reclaimed by a heap with the same tag (using %u instead)\n", page->heap_tag, heap->tag ); + } + mi_page_set_heap(page, target_heap); _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) _mi_page_free_collect(page, false); // ensure used count is up to date if (mi_page_all_free(page)) { @@ -907,8 +914,8 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, } else { // otherwise reclaim it into the heap - _mi_page_reclaim(heap, page); - if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page)) { + _mi_page_reclaim(target_heap, page); + if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page) && heap == target_heap) { if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; } } } From 7ea7296d3e71940cd11488d364a9c8a3618ecf63 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 19 May 2024 12:03:55 -0700 Subject: [PATCH 25/34] add malloc_good_size override maacro, see issue #900 and pr #901 --- include/mimalloc-override.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index c63b0b91..b79a6b3b 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -43,6 +43,7 @@ not accidentally mix pointers from different allocators). #define reallocf(p,n) mi_reallocf(p,n) #define malloc_size(p) mi_usable_size(p) #define malloc_usable_size(p) mi_usable_size(p) +#define malloc_good_size(sz) mi_malloc_good_size(sz) #define cfree(p) mi_free(p) #define valloc(n) mi_valloc(n) From 03224e17a01228e3cdb531343c925b772c5de567 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 19 May 2024 13:25:31 -0700 Subject: [PATCH 26/34] bump cmake required version to 3.18 and detect libatomic more reliably, see pr #898 --- CMakeLists.txt | 12 +++++------- include/mimalloc-override.h | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cc2fc46..08435964 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.18) project(libmimalloc C CXX) set(CMAKE_C_STANDARD 11) @@ -35,6 +35,7 @@ option(MI_NO_THP "Disable transparent huge pages support on Linux/And option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") @@ -354,12 +355,9 @@ else() list(APPEND mi_libraries ${MI_LIBRT}) set(pc_libraries "${pc_libraries} -lrt") endif() - find_library(MI_LIBATOMIC atomic) - if (NOT MI_LIBATOMIC AND MI_USE_LIBATOMIC) - set(MI_LIBATOMIC atomic) - endif() - if (MI_LIBATOMIC) - list(APPEND mi_libraries ${MI_LIBATOMIC}) + check_linker_flag(C "-latomic" MI_HAS_LIBATOMIC) # do not use `find_library` as libatomic is not always in the system path (issue #898) + if (MI_HAS_LIBATOMIC OR MI_USE_LIBATOMIC) + list(APPEND mi_libraries "atomic") set(pc_libraries "${pc_libraries} -latomic") endif() endif() diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index b79a6b3b..48a8a622 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -24,7 +24,7 @@ not accidentally mix pointers from different allocators). #define free(p) mi_free(p) #define strdup(s) mi_strdup(s) -#define strndup(s,n) mi_strndup(s,n) +#define strndup(s,n) mi_strndup(s,n) #define realpath(f,n) mi_realpath(f,n) // Microsoft extensions From f739e4f2c355ca652c72612263a2ccb9829d9e6a Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 19 May 2024 15:38:26 -0700 Subject: [PATCH 27/34] check all link libraries (pthread, rt, and atomic) via check_linker_flag first before using find_library (see also issue #898 --- CMakeLists.txt | 56 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 08435964..866693eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,25 +340,40 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) endif() # extra needed libraries + +# we prefer -l test over `find_library` as sometimes core libraries +# like `libatomic` are not on the system path (see issue #898) +function(add_link_library libname outlibname) + check_linker_flag(C "-l${libname}" mi_has_lib${libname}) + if (mi_has_lib${libname}) + message(VERBOSE "link library: -l${libname}") + set(${outlibname} ${libname} PARENT_SCOPE) + else() + find_library(MI_LIBPATH libname) + if (MI_LIBPATH) + message(VERBOSE "link library ${libname} at ${MI_LIBPATH}") + set(${outlibname} ${MI_LIBPATH} PARENT_SCOPE) + else() + message(VERBOSE "link library not found: ${libname}") + set(${outlibname} "" PARENT_SCOPE) + endif() + endif() +endfunction() + if(WIN32) - list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) - set(pc_libraries "-lpsapi -lshell32 -luser32 -ladvapi32 -lbcrypt") + list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() - set(pc_libraries "") - find_library(MI_LIBPTHREAD pthread) - if (MI_LIBPTHREAD) - list(APPEND mi_libraries ${MI_LIBPTHREAD}) - set(pc_libraries "${pc_libraries} -pthread") + add_link_library("pthread" MI_LIB_PTHREAD) + if(MI_LIB_PTHREAD) + list(APPEND mi_libraries "${MI_LIB_PTHREAD}") endif() - find_library(MI_LIBRT rt) - if(MI_LIBRT) - list(APPEND mi_libraries ${MI_LIBRT}) - set(pc_libraries "${pc_libraries} -lrt") + add_link_library("rt" MI_LIB_RT) + if(MI_LIB_RT) + list(APPEND mi_libraries "${MI_LIB_RT}") endif() - check_linker_flag(C "-latomic" MI_HAS_LIBATOMIC) # do not use `find_library` as libatomic is not always in the system path (issue #898) - if (MI_HAS_LIBATOMIC OR MI_USE_LIBATOMIC) - list(APPEND mi_libraries "atomic") - set(pc_libraries "${pc_libraries} -latomic") + add_link_library("atomic" MI_LIB_ATOMIC) + if(MI_LIB_ATOMIC) + list(APPEND mi_libraries "${MI_LIB_ATOMIC}") endif() endif() @@ -520,6 +535,15 @@ if (MI_BUILD_OBJECT) endif() # pkg-config file support +set(pc_libraries "") +foreach(item IN LISTS mi_libraries) + if(item MATCHES " *[-].*") + set(pc_libraries "${pc_libraries} ${item}") + else() + set(pc_libraries "${pc_libraries} -l${item}") + endif() +endforeach() + include("cmake/JoinPaths.cmake") join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") join_paths(libdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_LIBDIR}") @@ -528,6 +552,8 @@ configure_file(mimalloc.pc.in mimalloc.pc @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mimalloc.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/") + + # ----------------------------------------------------------------------------- # API surface testing # ----------------------------------------------------------------------------- From 9cd7e4cc68c1f518fa4ac27ab2de43a2b7a608e7 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 19 May 2024 15:43:15 -0700 Subject: [PATCH 28/34] rename library check to find_link_library --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 866693eb..6bac3b94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,7 @@ endif() # we prefer -l test over `find_library` as sometimes core libraries # like `libatomic` are not on the system path (see issue #898) -function(add_link_library libname outlibname) +function(find_link_library libname outlibname) check_linker_flag(C "-l${libname}" mi_has_lib${libname}) if (mi_has_lib${libname}) message(VERBOSE "link library: -l${libname}") @@ -363,15 +363,15 @@ endfunction() if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() - add_link_library("pthread" MI_LIB_PTHREAD) + find_link_library("pthread" MI_LIB_PTHREAD) if(MI_LIB_PTHREAD) list(APPEND mi_libraries "${MI_LIB_PTHREAD}") endif() - add_link_library("rt" MI_LIB_RT) + find_link_library("rt" MI_LIB_RT) if(MI_LIB_RT) list(APPEND mi_libraries "${MI_LIB_RT}") endif() - add_link_library("atomic" MI_LIB_ATOMIC) + find_link_library("atomic" MI_LIB_ATOMIC) if(MI_LIB_ATOMIC) list(APPEND mi_libraries "${MI_LIB_ATOMIC}") endif() From b9b321d3284b517344ea71fb49830d953b5f4be9 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 19 May 2024 20:42:28 -0700 Subject: [PATCH 29/34] use _builtin_thread_pointer also on gcc 11-x64 and clang-14-x64 --- include/mimalloc/prim.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 126ea7cd..3f4574dd 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -198,7 +198,7 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce tcb[slot] = value; #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 MI_UNUSED(ofs); - pthread_setspecific(slot, value); + pthread_setspecific(slot, value); #endif } @@ -208,13 +208,18 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce // but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) // Nevertheless, it seems needed on older graviton platforms (see issue #851). // For now, we only enable this for specific platforms. -#if defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__) /* special case aarch64 for older gcc versions (issue #851) */ \ - && !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \ +#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \ + && !defined(MI_LIBC_MUSL) \ && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */ -#define MI_USE_BUILTIN_THREAD_POINTER 1 + #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ + || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__))) + #define MI_USE_BUILTIN_THREAD_POINTER 1 + #endif #endif + // defined in `init.c`; do not use these directly extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from extern bool _mi_process_is_initialized; // has mi_process_init been called? @@ -239,11 +244,11 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { return (uintptr_t)NtCurrentTeb(); } -#elif MI_USE_BUILTIN_THREAD_POINTER +#elif MI_USE_BUILTIN_THREAD_POINTER static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { // Works on most Unix based platforms with recent compilers - return (uintptr_t)__builtin_thread_pointer(); + return (uintptr_t)__builtin_thread_pointer(); } #elif defined(MI_HAS_TLS_SLOT) From d5ac4f7b95dfdd886ae69cd8798e8defbbefd664 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 19 May 2024 20:42:58 -0700 Subject: [PATCH 30/34] add example docker file for testing on manylinux-x64 --- docker/manylinux-x64/Dockerfile | 22 ++++++++++++++++++++++ docker/readme.md | 10 ++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docker/manylinux-x64/Dockerfile create mode 100644 docker/readme.md diff --git a/docker/manylinux-x64/Dockerfile b/docker/manylinux-x64/Dockerfile new file mode 100644 index 00000000..566685d4 --- /dev/null +++ b/docker/manylinux-x64/Dockerfile @@ -0,0 +1,22 @@ +FROM quay.io/pypa/manylinux2014_x86_64 + +# Install tools +RUN yum install -y openssl-devel +RUN yum install -y gcc gcc-c++ kernel-devel make +RUN yum install -y git cmake +RUN yum install -y vim + +RUN mkdir -p /home/dev +WORKDIR /home/dev + +# Get mimalloc +RUN git clone https://github.com/microsoft/mimalloc -b dev-slice +RUN mkdir -p mimalloc/out/release +RUN mkdir -p mimalloc/out/debug + +# Build mimalloc debug +WORKDIR /home/dev/mimalloc/out/debug +RUN cmake ../.. -DMI_DEBUG_FULL=ON +RUN make -j + +CMD ["/bin/sh"] \ No newline at end of file diff --git a/docker/readme.md b/docker/readme.md new file mode 100644 index 00000000..b3d90094 --- /dev/null +++ b/docker/readme.md @@ -0,0 +1,10 @@ +Various example docker files used for testing. + +Usage: + +``` +> cd +> docker build -t -mimalloc . +> docker run -it -mimalloc +>> make test +``` From 836ee0a94c568eab171ab63ab655c7b64fa10c98 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 19 May 2024 21:32:22 -0700 Subject: [PATCH 31/34] add alpine docker files --- .gitignore | 2 ++ docker/alpine-arm32v7/Dockerfile | 28 ++++++++++++++++++++++++++++ docker/alpine/Dockerfile | 23 +++++++++++++++++++++++ docker/manylinux-x64/Dockerfile | 1 + 4 files changed, 54 insertions(+) create mode 100644 docker/alpine-arm32v7/Dockerfile create mode 100644 docker/alpine/Dockerfile diff --git a/.gitignore b/.gitignore index f8b7f5eb..df1d58eb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ ide/vs20??/VTune* out/ docs/ *.zip +*.tar +*.gz diff --git a/docker/alpine-arm32v7/Dockerfile b/docker/alpine-arm32v7/Dockerfile new file mode 100644 index 00000000..56f071db --- /dev/null +++ b/docker/alpine-arm32v7/Dockerfile @@ -0,0 +1,28 @@ +# install from an image +# download first an appropiate tar.gz image into the current directory +# from: +FROM scratch + +# Substitute the image name that was downloaded +ADD alpine-minirootfs-20240329-armv7.tar.gz / + +# Install tools +RUN apk add build-base make cmake +RUN apk add git +RUN apk add vim + +RUN mkdir -p /home/dev +WORKDIR /home/dev + +# Get mimalloc +RUN git clone https://github.com/microsoft/mimalloc -b dev-slice +RUN mkdir -p mimalloc/out/release +RUN mkdir -p mimalloc/out/debug + +# Build mimalloc debug +WORKDIR /home/dev/mimalloc/out/debug +RUN cmake ../.. -DMI_DEBUG_FULL=ON +RUN make -j +RUN make test + +CMD ["/bin/sh"] diff --git a/docker/alpine/Dockerfile b/docker/alpine/Dockerfile new file mode 100644 index 00000000..b222b791 --- /dev/null +++ b/docker/alpine/Dockerfile @@ -0,0 +1,23 @@ +# alpine image +FROM alpine + +# Install tools +RUN apk add build-base make cmake +RUN apk add git +RUN apk add vim + +RUN mkdir -p /home/dev +WORKDIR /home/dev + +# Get mimalloc +RUN git clone https://github.com/microsoft/mimalloc -b dev-slice +RUN mkdir -p mimalloc/out/release +RUN mkdir -p mimalloc/out/debug + +# Build mimalloc debug +WORKDIR /home/dev/mimalloc/out/debug +RUN cmake ../.. -DMI_DEBUG_FULL=ON +RUN make -j +RUN make test + +CMD ["/bin/sh"] \ No newline at end of file diff --git a/docker/manylinux-x64/Dockerfile b/docker/manylinux-x64/Dockerfile index 566685d4..22d37e5a 100644 --- a/docker/manylinux-x64/Dockerfile +++ b/docker/manylinux-x64/Dockerfile @@ -18,5 +18,6 @@ RUN mkdir -p mimalloc/out/debug WORKDIR /home/dev/mimalloc/out/debug RUN cmake ../.. -DMI_DEBUG_FULL=ON RUN make -j +RUN make test CMD ["/bin/sh"] \ No newline at end of file From bd04cec1c45595cd048b434fa35258002952b4e5 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 21 May 2024 11:57:52 -0700 Subject: [PATCH 32/34] roll back export ignore on the bin directory as it needed by vcpkg (et al) to build on Windows --- .gitattributes | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index f083b107..0332e031 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,4 +10,3 @@ *.dll binary *.lib binary *.exe binary -bin export-ignore From 4aa97336f7688db69cc20bbc209ebceed0439a6d Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 21 May 2024 12:06:52 -0700 Subject: [PATCH 33/34] remove the need for cmake patch in vcpkg port --- CMakeLists.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bac3b94..bcfe91d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -339,6 +339,10 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) list(APPEND mi_cflags /Zc:__cplusplus) endif() +if(MINGW) + add_definitions(-D_WIN32_WINNT=0x600) +endif() + # extra needed libraries # we prefer -l test over `find_library` as sometimes core libraries @@ -382,7 +386,8 @@ endif() # ----------------------------------------------------------------------------- # dynamic/shared library and symlinks always go to /usr/local/lib equivalent -set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") +set(mi_install_bindir "${CMAKE_INSTALL_BINDIR}") # static libraries and object files, includes, and cmake config files # are either installed at top level, or use versioned directories for side-by-side installation (default) @@ -466,10 +471,10 @@ if(MI_BUILD_SHARED) add_custom_command(TARGET mimalloc POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" $ COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory") - install(FILES "$/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_libdir}) + install(FILES "$/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_bindir}) endif() - install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) + install(TARGETS mimalloc EXPORT mimalloc ARCHIVE DESTINATION ${mi_install_libdir} RUNTIME DESTINATION ${mi_install_bindir} LIBRARY DESTINATION ${mi_install_libdir}) install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir}) endif() From 1f3ecec2e589deabad4fed62853b8bca53ec00e3 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 21 May 2024 12:25:13 -0700 Subject: [PATCH 34/34] update readme --- readme.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 91974587..a0296b43 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.1.6` (2024-05-13). -Latest v1 tag: `v1.8.6` (2024-05-13). +Latest release tag: `v2.1.7` (2024-05-21). +Latest v1 tag: `v1.8.7` (2024-05-21). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -82,6 +82,8 @@ memory usage and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. +* 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches + from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches. * 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation. * 2024-04-22, `v1.8.4`, `v2.1.4`: Fixes various bugs and build issues. Add `MI_LIBC_MUSL` cmake flag for musl builds. Free-ing code is refactored into a separate module (`free.c`). Mimalloc page info is simplified with the block size