mirrors
/
mimalloc
mirror of https://github.com/microsoft/mimalloc
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev-slice'

This commit is contained in:
Daan 2024-05-21 12:27:13 -07:00
parent a1b284de0a 8c532c32c3
commit 2765ec9302
26 changed files with 388 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored
View File

@ -10,4 +10,3 @@
*.dll binary
*.lib binary
*.exe binary
bin export-ignore

2
.gitignore vendored
View File

@ -7,3 +7,5 @@ ide/vs20??/VTune*
out/
docs/
*.zip
*.tar
*.gz

71
CMakeLists.txt
View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.13)
cmake_minimum_required(VERSION 3.18)
project(libmimalloc C CXX)
set(CMAKE_C_STANDARD 11)
@ -35,6 +35,7 @@ option(MI_NO_THP "Disable transparent huge pages support on Linux/And
option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF)
option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF)
include(CheckLinkerFlag) # requires cmake 3.18
include(CheckIncludeFiles)
include(GNUInstallDirs)
include("cmake/mimalloc-config-version.cmake")
@ -338,29 +339,45 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914)
list(APPEND mi_cflags /Zc:__cplusplus)
endif()
if(MINGW)
add_definitions(-D_WIN32_WINNT=0x600)
endif()
# extra needed libraries
# we prefer -l<lib> test over `find_library` as sometimes core libraries
# like `libatomic` are not on the system path (see issue #898)
function(find_link_library libname outlibname)
check_linker_flag(C "-l${libname}" mi_has_lib${libname})
if (mi_has_lib${libname})
message(VERBOSE "link library: -l${libname}")
set(${outlibname} ${libname} PARENT_SCOPE)
else()
find_library(MI_LIBPATH libname)
if (MI_LIBPATH)
message(VERBOSE "link library ${libname} at ${MI_LIBPATH}")
set(${outlibname} ${MI_LIBPATH} PARENT_SCOPE)
else()
message(VERBOSE "link library not found: ${libname}")
set(${outlibname} "" PARENT_SCOPE)
endif()
endif()
endfunction()
if(WIN32)
list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt)
set(pc_libraries "-lpsapi -lshell32 -luser32 -ladvapi32 -lbcrypt")
else()
set(pc_libraries "")
find_library(MI_LIBPTHREAD pthread)
if (MI_LIBPTHREAD)
list(APPEND mi_libraries ${MI_LIBPTHREAD})
set(pc_libraries "${pc_libraries} -pthread")
find_link_library("pthread" MI_LIB_PTHREAD)
if(MI_LIB_PTHREAD)
list(APPEND mi_libraries "${MI_LIB_PTHREAD}")
endif()
find_library(MI_LIBRT rt)
if(MI_LIBRT)
list(APPEND mi_libraries ${MI_LIBRT})
set(pc_libraries "${pc_libraries} -lrt")
find_link_library("rt" MI_LIB_RT)
if(MI_LIB_RT)
list(APPEND mi_libraries "${MI_LIB_RT}")
endif()
find_library(MI_LIBATOMIC atomic)
if (NOT MI_LIBATOMIC AND MI_USE_LIBATOMIC)
set(MI_LIBATOMIC atomic)
endif()
if (MI_LIBATOMIC)
list(APPEND mi_libraries ${MI_LIBATOMIC})
set(pc_libraries "${pc_libraries} -latomic")
find_link_library("atomic" MI_LIB_ATOMIC)
if(MI_LIB_ATOMIC)
list(APPEND mi_libraries "${MI_LIB_ATOMIC}")
endif()
endif()
@ -369,7 +386,8 @@ endif()
# -----------------------------------------------------------------------------
# dynamic/shared library and symlinks always go to /usr/local/lib equivalent
set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}")
set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}")
set(mi_install_bindir "${CMAKE_INSTALL_BINDIR}")
# static libraries and object files, includes, and cmake config files
# are either installed at top level, or use versioned directories for side-by-side installation (default)
@ -453,10 +471,10 @@ if(MI_BUILD_SHARED)
add_custom_command(TARGET mimalloc POST_BUILD
COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" $<TARGET_FILE_DIR:mimalloc>
COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory")
install(FILES "$<TARGET_FILE_DIR:mimalloc>/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_libdir})
install(FILES "$<TARGET_FILE_DIR:mimalloc>/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_bindir})
endif()
install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY)
install(TARGETS mimalloc EXPORT mimalloc ARCHIVE DESTINATION ${mi_install_libdir} RUNTIME DESTINATION ${mi_install_bindir} LIBRARY DESTINATION ${mi_install_libdir})
install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir})
endif()
@ -522,6 +540,15 @@ if (MI_BUILD_OBJECT)
endif()
# pkg-config file support
set(pc_libraries "")
foreach(item IN LISTS mi_libraries)
if(item MATCHES " *[-].*")
set(pc_libraries "${pc_libraries} ${item}")
else()
set(pc_libraries "${pc_libraries} -l${item}")
endif()
endforeach()
include("cmake/JoinPaths.cmake")
join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
join_paths(libdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_LIBDIR}")
@ -530,6 +557,8 @@ configure_file(mimalloc.pc.in mimalloc.pc @ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mimalloc.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
# -----------------------------------------------------------------------------
# API surface testing
# -----------------------------------------------------------------------------

2
cmake/mimalloc-config-version.cmake
View File

@ -1,6 +1,6 @@
set(mi_version_major 2)
set(mi_version_minor 1)
set(mi_version_patch 6)
set(mi_version_patch 7)
set(mi_version ${mi_version_major}.${mi_version_minor})
set(PACKAGE_VERSION ${mi_version})

28
docker/alpine-arm32v7/Dockerfile Normal file
View File

@ -0,0 +1,28 @@
# install from an image
# download first an appropiate tar.gz image into the current directory
# from: <https://github.com/alpinelinux/docker-alpine/tree/edge/armv7>
FROM scratch
# Substitute the image name that was downloaded
ADD alpine-minirootfs-20240329-armv7.tar.gz /
# Install tools
RUN apk add build-base make cmake
RUN apk add git
RUN apk add vim
RUN mkdir -p /home/dev
WORKDIR /home/dev
# Get mimalloc
RUN git clone https://github.com/microsoft/mimalloc -b dev-slice
RUN mkdir -p mimalloc/out/release
RUN mkdir -p mimalloc/out/debug
# Build mimalloc debug
WORKDIR /home/dev/mimalloc/out/debug
RUN cmake ../.. -DMI_DEBUG_FULL=ON
RUN make -j
RUN make test
CMD ["/bin/sh"]

23
docker/alpine/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
# alpine image
FROM alpine
# Install tools
RUN apk add build-base make cmake
RUN apk add git
RUN apk add vim
RUN mkdir -p /home/dev
WORKDIR /home/dev
# Get mimalloc
RUN git clone https://github.com/microsoft/mimalloc -b dev-slice
RUN mkdir -p mimalloc/out/release
RUN mkdir -p mimalloc/out/debug
# Build mimalloc debug
WORKDIR /home/dev/mimalloc/out/debug
RUN cmake ../.. -DMI_DEBUG_FULL=ON
RUN make -j
RUN make test
CMD ["/bin/sh"]

23
docker/manylinux-x64/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
FROM quay.io/pypa/manylinux2014_x86_64
# Install tools
RUN yum install -y openssl-devel
RUN yum install -y gcc gcc-c++ kernel-devel make
RUN yum install -y git cmake
RUN yum install -y vim
RUN mkdir -p /home/dev
WORKDIR /home/dev
# Get mimalloc
RUN git clone https://github.com/microsoft/mimalloc -b dev-slice
RUN mkdir -p mimalloc/out/release
RUN mkdir -p mimalloc/out/debug
# Build mimalloc debug
WORKDIR /home/dev/mimalloc/out/debug
RUN cmake ../.. -DMI_DEBUG_FULL=ON
RUN make -j
RUN make test
CMD ["/bin/sh"]

10
docker/readme.md Normal file
View File

@ -0,0 +1,10 @@
Various example docker files used for testing.
Usage:
```
> cd <host>
> docker build -t <host>-mimalloc .
> docker run -it <host>-mimalloc
>> make test
```

3
include/mimalloc-override.h
View File

@ -24,7 +24,7 @@ not accidentally mix pointers from different allocators).
#define free(p) mi_free(p)
#define strdup(s) mi_strdup(s)
#define strndup(s,n) mi_strndup(s,n)
#define strndup(s,n) mi_strndup(s,n)
#define realpath(f,n) mi_realpath(f,n)
// Microsoft extensions
@ -43,6 +43,7 @@ not accidentally mix pointers from different allocators).
#define reallocf(p,n) mi_reallocf(p,n)
#define malloc_size(p) mi_usable_size(p)
#define malloc_usable_size(p) mi_usable_size(p)
#define malloc_good_size(sz) mi_malloc_good_size(sz)
#define cfree(p) mi_free(p)
#define valloc(n) mi_valloc(n)

7
include/mimalloc.h
View File

@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MIMALLOC_H
#define MIMALLOC_H
#define MI_MALLOC_VERSION 216 // major + 2 digits minor
#define MI_MALLOC_VERSION 217 // major + 2 digits minor
// ------------------------------------------------------
// Compiler specific attributes
@ -328,7 +328,7 @@ typedef enum mi_option_e {
mi_option_allow_large_os_pages, // allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process.
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup
mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)
mi_option_deprecated_segment_cache,
mi_option_deprecated_page_reset,
mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination
@ -342,11 +342,12 @@ typedef enum mi_option_e {
mi_option_max_warnings, // issue at most N warning messages
mi_option_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%)
mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
mi_option_arena_reserve, // initial memory size in KiB for arena reservation (= 1 GiB on 64-bit)
mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)
mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10)
mi_option_purge_extend_delay,
mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1)
mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)
mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
_mi_option_last,
// legacy option names
mi_option_large_os_pages = mi_option_allow_large_os_pages,

4
include/mimalloc/atomic.h
View File

@ -132,7 +132,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
#elif defined(_MSC_VER)
// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics.
// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
@ -201,7 +201,7 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_
#else
uintptr_t x = *p;
if (mo > mi_memory_order_relaxed) {
while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
}
return x;
#endif

12
include/mimalloc/internal.h
View File

@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file
// functions and macros.
// --------------------------------------------------------------------------
#include "mimalloc/types.h"
#include "mimalloc/track.h"
#include "types.h"
#include "track.h"
#if (MI_DEBUG>0)
#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)
@ -88,6 +88,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
void _mi_thread_done(mi_heap_t* heap);
void _mi_thread_data_collect(void);
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
// os.c
void _mi_os_init(void); // called from process init
@ -186,11 +187,13 @@ size_t _mi_bin_size(uint8_t bin); // for stats
uint8_t _mi_bin(size_t size); // for stats
// "heap.c"
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag);
void _mi_heap_destroy_pages(mi_heap_t* heap);
void _mi_heap_collect_abandon(mi_heap_t* heap);
void _mi_heap_set_default_direct(mi_heap_t* heap);
bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid);
void _mi_heap_unsafe_destroy_all(void);
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag);
// "stats.c"
void _mi_stats_done(mi_stats_t* stats);
@ -379,10 +382,10 @@ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
}
#else /* __builtin_umul_overflow is unavailable */
static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
#define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
#define MI_MUL_COULD_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
*total = count * size;
// note: gcc/clang optimize this to directly check the overflow flag
return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count);
return ((size >= MI_MUL_COULD_OVERFLOW || count >= MI_MUL_COULD_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count);
}
#endif
@ -546,6 +549,7 @@ static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {
mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING);
mi_atomic_store_release(&page->xheap,(uintptr_t)heap);
if (heap != NULL) { page->heap_tag = heap->tag; }
}
// Thread free flag helpers

21
include/mimalloc/prim.h
View File

@ -26,7 +26,7 @@ typedef struct mi_os_mem_config_s {
size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)
size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)
bool has_overcommit; // can we reserve more memory than can be actually committed?
bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc)
bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
} mi_os_mem_config_t;
@ -208,13 +208,18 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce
// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
// Nevertheless, it seems needed on older graviton platforms (see issue #851).
// For now, we only enable this for specific platforms.
#if defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__) /* special case aarch64 for older gcc versions (issue #851) */ \
&& !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
&& !defined(MI_LIBC_MUSL) \
&& (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
#define MI_USE_BUILTIN_THREAD_POINTER 1
#if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
|| (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
|| (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
#define MI_USE_BUILTIN_THREAD_POINTER 1
#endif
#endif
// defined in `init.c`; do not use these directly
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
extern bool _mi_process_is_initialized; // has mi_process_init been called?
@ -222,7 +227,13 @@ extern bool _mi_process_is_initialized; // has mi_process_init been
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept;
// Get a unique id for the current thread.
#if defined(_WIN32)
#if defined(MI_PRIM_THREAD_ID)
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
}
#elif defined(_WIN32)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN

4
include/mimalloc/types.h
View File

@ -24,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stddef.h> // ptrdiff_t
#include <stdint.h> // uintptr_t, uint16_t, etc
#include "mimalloc/atomic.h" // _Atomic
#include "atomic.h" // _Atomic
#ifdef _MSC_VER
#pragma warning(disable:4214) // bitfield is not int
@ -319,6 +319,7 @@ typedef struct mi_page_s {
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
uint16_t used; // number of blocks in use (including blocks in `thread_free`)
uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`)
uint8_t heap_tag; // tag of the owning heap, used for separated heaps by object type
// padding
size_t block_size; // size available in each block (always `>0`)
uint8_t* page_start; // start of the page area containing the blocks
@ -538,6 +539,7 @@ struct mi_heap_s {
size_t page_retired_max; // largest retired index into the `pages` array.
mi_heap_t* next; // list of heaps per thread
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
uint8_t tag; // custom tag, can be used for separating heaps based on the object types
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
};

6
readme.md
View File

@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac
Initially developed by Daan Leijen for the runtime systems of the
[Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages.
Latest release tag: `v2.1.6` (2024-05-13).
Latest v1 tag: `v1.8.6` (2024-05-13).
Latest release tag: `v2.1.7` (2024-05-21).
Latest v1 tag: `v1.8.7` (2024-05-21).
mimalloc is a drop-in replacement for `malloc` and can be used in other programs
without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as:
@ -82,6 +82,8 @@ memory usage
and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance
(see [below](#performance)); please report if you observe any significant performance regression.
* 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches
from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches.
* 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation.
* 2024-04-22, `v1.8.4`, `v2.1.4`: Fixes various bugs and build issues. Add `MI_LIBC_MUSL` cmake flag for musl builds.
Free-ing code is refactored into a separate module (`free.c`). Mimalloc page info is simplified with the block size

14
src/arena.c
View File

@ -57,6 +57,7 @@ typedef struct mi_arena_s {
mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted)
mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here)
mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)
// do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields.
} mi_arena_t;
@ -144,18 +145,19 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit
#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit
static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX];
static _Atomic(size_t) mi_arena_static_top;
static mi_decl_cache_align uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; // must be cache aligned, see issue #895
static mi_decl_cache_align _Atomic(size_t) mi_arena_static_top;
static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (size == 0 || size > MI_ARENA_STATIC_MAX) return NULL;
if ((mi_atomic_load_relaxed(&mi_arena_static_top) + size) > MI_ARENA_STATIC_MAX) return NULL;
const size_t toplow = mi_atomic_load_relaxed(&mi_arena_static_top);
if ((toplow + size) > MI_ARENA_STATIC_MAX) return NULL;
// try to claim space
if (alignment == 0) { alignment = 1; }
if (alignment < MI_MAX_ALIGN_SIZE) { alignment = MI_MAX_ALIGN_SIZE; }
const size_t oversize = size + alignment - 1;
if (oversize > MI_ARENA_STATIC_MAX) return NULL;
if (toplow + oversize > MI_ARENA_STATIC_MAX) return NULL;
const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize);
size_t top = oldtop + oversize;
if (top > MI_ARENA_STATIC_MAX) {
@ -169,7 +171,7 @@ static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* m
memid->initially_zero = true;
const size_t start = _mi_align_up(oldtop, alignment);
uint8_t* const p = &mi_arena_static[start];
_mi_memzero(p, size);
_mi_memzero_aligned(p, size);
return p;
}

54
src/heap.c
View File

@ -128,6 +128,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
const bool force = (collect >= MI_FORCE);
_mi_deferred_free(heap, force);
// python/cpython#112532: we may be called from a thread that is not the owner of the heap
const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id());
// note: never reclaim on collect but leave it to threads that need storage to reclaim
const bool force_main =
#ifdef NDEBUG
@ -135,7 +138,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
#else
collect >= MI_FORCE
#endif
&& _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim;
&& is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim;
if (force_main) {
// the main thread is abandoned (end-of-program), try to reclaim all abandoned segments.
@ -164,7 +167,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
_mi_abandoned_collect(heap, collect == MI_FORCE /* force? */, &heap->tld->segments);
// if forced, collect thread data cache on program-exit (or shared library unload)
if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
if (force && is_main_thread && mi_heap_is_backing(heap)) {
_mi_thread_data_collect(); // collect thread data cache
}
@ -208,22 +211,33 @@ mi_heap_t* mi_heap_get_backing(void) {
return bheap;
}
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag) {
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->tld = tld;
heap->thread_id = _mi_thread_id();
heap->arena_id = arena_id;
heap->no_reclaim = noreclaim;
heap->tag = tag;
if (heap == tld->heap_backing) {
_mi_random_init(&heap->random);
}
else {
_mi_random_split(&tld->heap_backing->random, &heap->random);
}
heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
// push on the thread local heaps list
heap->next = heap->tld->heaps;
heap->tld->heaps = heap;
}
mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) {
mi_heap_t* bheap = mi_heap_get_backing();
mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
if (heap == NULL) return NULL;
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->tld = bheap->tld;
heap->thread_id = _mi_thread_id();
heap->arena_id = arena_id;
_mi_random_split(&bheap->random, &heap->random);
heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe
// push on the thread local heaps list
heap->next = heap->tld->heaps;
heap->tld->heaps = heap;
// don't reclaim abandoned pages or otherwise destroy is unsafe
_mi_heap_init(heap, bheap->tld, arena_id, true /* no reclaim */, 0 /* default tag */);
return heap;
}
@ -281,6 +295,18 @@ static void mi_heap_free(mi_heap_t* heap) {
mi_free(heap);
}
// return a heap on the same thread as `heap` specialized for the specified tag (if it exists)
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) {
if (heap->tag == tag) {
return heap;
}
for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) {
if (curr->tag == tag) {
return curr;
}
}
return NULL;
}
/* -----------------------------------------------------------
Heap destroy

40
src/init.c
View File

@ -25,6 +25,7 @@ const mi_page_t _mi_page_empty = {
NULL, // local_free
0, // used
0, // block size shift
0, // heap tag
0, // block_size
NULL, // page_start
#if (MI_PADDING || MI_ENCODE_FREELIST)
@ -33,9 +34,7 @@ const mi_page_t _mi_page_empty = {
MI_ATOMIC_VAR_INIT(0), // xthread_free
MI_ATOMIC_VAR_INIT(0), // xheap
NULL, NULL
#if MI_INTPTR_SIZE==8
, { 0 } // padding
#endif
};
#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
@ -124,7 +123,8 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
0, // page count
MI_BIN_FULL, 0, // page retired min/max
NULL, // next
false,
false, // can reclaim
0, // tag
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY
};
@ -170,6 +170,7 @@ mi_heap_t _mi_heap_main = {
MI_BIN_FULL, 0, // page retired min/max
NULL, // next heap
false, // can reclaim
0, // tag
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY
};
@ -288,7 +289,7 @@ void _mi_thread_data_collect(void) {
}
// Initialize the thread local default heap, called from `mi_thread_init`
static bool _mi_heap_init(void) {
static bool _mi_thread_heap_init(void) {
if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true;
if (_mi_is_main_thread()) {
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization
@ -304,26 +305,25 @@ static bool _mi_heap_init(void) {
mi_tld_t* tld = &td->tld;
mi_heap_t* heap = &td->heap;
_mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld));
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap));
heap->thread_id = _mi_thread_id();
_mi_random_init(&heap->random);
heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
heap->tld = tld;
tld->heap_backing = heap;
tld->heaps = heap;
tld->segments.stats = &tld->stats;
tld->segments.os = &tld->os;
tld->os.stats = &tld->stats;
_mi_tld_init(tld, heap); // must be before `_mi_heap_init`
_mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */);
_mi_heap_set_default_direct(heap);
}
return false;
}
// initialize thread local data
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) {
_mi_memcpy_aligned(tld, &tld_empty, sizeof(mi_tld_t));
tld->heap_backing = bheap;
tld->heaps = NULL;
tld->segments.stats = &tld->stats;
tld->segments.os = &tld->os;
tld->os.stats = &tld->stats;
}
// Free the thread local default heap (called from `mi_thread_done`)
static bool _mi_heap_done(mi_heap_t* heap) {
static bool _mi_thread_heap_done(mi_heap_t* heap) {
if (!mi_heap_is_initialized(heap)) return true;
// reset default heap
@ -420,7 +420,7 @@ void mi_thread_init(void) mi_attr_noexcept
// initialize the thread local default heap
// (this will call `_mi_heap_set_default_direct` and thus set the
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
if (_mi_heap_init()) return; // returns true if already initialized
if (_mi_thread_heap_init()) return; // returns true if already initialized
_mi_stat_increase(&_mi_stats_main.threads, 1);
mi_atomic_increment_relaxed(&thread_count);
@ -452,7 +452,7 @@ void _mi_thread_done(mi_heap_t* heap)
if (heap->thread_id != _mi_thread_id()) return;
// abandon the thread local heap
if (_mi_heap_done(heap)) return; // returns true if already ran
if (_mi_thread_heap_done(heap)) return; // returns true if already ran
}
void _mi_heap_set_default_direct(mi_heap_t* heap) {

41
src/options.c
View File

@ -65,7 +65,7 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
{-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
{ 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve OS memory in advance
{ 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)
{ 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread
{ 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free
{ 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_purge,abandoned_page_reset) }, // reset free page memory when a thread terminates
@ -79,19 +79,20 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
{ 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output
{ 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
{ 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments per try.
{ 32, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output
{ 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
{ 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.
{ 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
#if (MI_INTPTR_SIZE>4)
{ 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time
{ 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
#else
{ 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) },
{ 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit
#endif
{ 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
{ 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) },
{ 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free
{ 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
{ 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
};
static void mi_option_init(mi_option_desc_t* desc);
@ -135,8 +136,12 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma
mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
mi_assert_internal(mi_option_has_size_in_kib(option));
long x = mi_option_get(option);
return (x < 0 ? 0 : (size_t)x * MI_KiB);
const long x = mi_option_get(option);
size_t size = (x < 0 ? 0 : (size_t)x);
if (mi_option_has_size_in_kib(option)) {
size *= MI_KiB;
}
return size;
}
void mi_option_set(mi_option_t option, long value) {
@ -479,14 +484,20 @@ static void mi_option_init(mi_option_desc_t* desc) {
else {
char* end = buf;
long value = strtol(buf, &end, 10);
if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_arena_reserve) {
// this option is interpreted in KiB to prevent overflow of `long`
if (mi_option_has_size_in_kib(desc->option)) {
// this option is interpreted in KiB to prevent overflow of `long` for large allocations
// (long is 32-bit on 64-bit windows, which allows for 4TiB max.)
size_t size = (value < 0 ? 0 : (size_t)value);
bool overflow = false;
if (*end == 'K') { end++; }
else if (*end == 'M') { value *= MI_KiB; end++; }
else if (*end == 'G') { value *= MI_MiB; end++; }
else { value = (value + MI_KiB - 1) / MI_KiB; }
if (end[0] == 'I' && end[1] == 'B') { end += 2; }
else if (*end == 'B') { end++; }
else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }
else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }
else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }
else { size = (size + MI_KiB - 1) / MI_KiB; }
if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB
else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb
if (overflow || size > MI_MAX_ALLOC_SIZE) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }
value = (size > LONG_MAX ? LONG_MAX : (long)size);
}
if (*end == 0) {
desc->value = value;

10
src/os.c
View File

@ -12,8 +12,6 @@ terms of the MIT license. A copy of the license can be found in the file
/* -----------------------------------------------------------
Initialization.
On windows initializes support for aligned allocation and
large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
----------------------------------------------------------- */
static mi_os_mem_config_t mi_os_mem_config = {
@ -21,7 +19,7 @@ static mi_os_mem_config_t mi_os_mem_config = {
0, // large page size (usually 2MiB)
4096, // allocation granularity
true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)
false, // must free whole? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
};
@ -239,7 +237,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment;
if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block
if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block
// over-allocate uncommitted (virtual) memory
p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats);
if (p == NULL) return NULL;
@ -260,7 +258,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area. (noop on sbrk)
// and selectively unmap parts around the over-allocated area.
void* aligned_p = mi_align_up_ptr(p, alignment);
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
@ -268,7 +266,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); }
// we can return the aligned pointer on `mmap` (and sbrk) systems
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
*base = aligned_p; // since we freed the pre part, `*base == p`.
}

2
src/prim/emscripten/prim.c
View File

@ -51,7 +51,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) {
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
config->alloc_granularity = 16;
config->has_overcommit = false;
config->must_free_whole = true;
config->has_partial_free = false;
config->has_virtual_reserve = false;
}

1
src/prim/osx/alloc-override-zone.c
View File

@ -422,6 +422,7 @@ __attribute__((constructor(0)))
#else
__attribute__((constructor)) // seems not supported by g++-11 on the M1
#endif
__attribute__((used))
static void _mi_macos_override_malloc(void) {
malloc_zone_t* purgeable_zone = NULL;

29
src/prim/unix/prim.c
View File

@ -57,7 +57,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <sys/sysctl.h>
#endif
#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(__OpenBSD__) && !defined(__sun)
#if defined(__linux__) || defined(__FreeBSD__)
#define MI_HAS_SYSCALL_H
#include <sys/syscall.h>
#endif
@ -67,37 +67,36 @@ terms of the MIT license. A copy of the license can be found in the file
// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
// and do allocation themselves; using syscalls prevents recursion when mimalloc is
// still initializing (issue #713)
// Declare inline to avoid unused function warnings.
//------------------------------------------------------------------------------------
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)
static int mi_prim_open(const char* fpath, int open_flags) {
static inline int mi_prim_open(const char* fpath, int open_flags) {
return syscall(SYS_open,fpath,open_flags,0);
}
static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
return syscall(SYS_read,fd,buf,bufsize);
}
static int mi_prim_close(int fd) {
static inline int mi_prim_close(int fd) {
return syscall(SYS_close,fd);
}
static int mi_prim_access(const char *fpath, int mode) {
static inline int mi_prim_access(const char *fpath, int mode) {
return syscall(SYS_access,fpath,mode);
}
#elif !defined(__sun) && \
(!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS and Solaris
#else
static int mi_prim_open(const char* fpath, int open_flags) {
static inline int mi_prim_open(const char* fpath, int open_flags) {
return open(fpath,open_flags);
}
static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
return read(fd,buf,bufsize);
}
static int mi_prim_close(int fd) {
static inline int mi_prim_close(int fd) {
return close(fd);
}
static int mi_prim_access(const char *fpath, int mode) {
static inline int mi_prim_access(const char *fpath, int mode) {
return access(fpath,mode);
}
@ -144,7 +143,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
}
config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this?
config->has_overcommit = unix_detect_overcommit();
config->must_free_whole = false; // mmap can free in parts
config->has_partial_free = true; // mmap can free in parts
config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)
// disable transparent huge pages for this process?
@ -199,7 +198,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0);
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
int err = errno;
_mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
_mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
}
if (p!=MAP_FAILED) return p;
// fall back to regular mmap
@ -224,7 +223,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
#else
int err = errno;
#endif
_mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
_mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
}
if (p!=MAP_FAILED) return p;
// fall back to regular mmap

2
src/prim/wasi/prim.c
View File

@ -23,7 +23,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
config->alloc_granularity = 16;
config->has_overcommit = false;
config->must_free_whole = true;
config->has_partial_free = false;
config->has_virtual_reserve = false;
}

47
src/prim/windows/prim.c
View File

@ -112,7 +112,7 @@ static bool win_enable_large_os_pages(size_t* large_page_size)
void _mi_prim_mem_init( mi_os_mem_config_t* config )
{
config->has_overcommit = false;
config->must_free_whole = true;
config->has_partial_free = false;
config->has_virtual_reserve = true;
// get the page size
SYSTEM_INFO si;
@ -178,7 +178,7 @@ int _mi_prim_free(void* addr, size_t size ) {
// VirtualAlloc
//---------------------------------------------
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {
#if (MI_INTPTR_SIZE >= 8)
// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
if (addr == NULL) {
@ -200,13 +200,53 @@ static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignmen
param.Arg.Pointer = &reqs;
void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, &param, 1);
if (p != NULL) return p;
_mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
_mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
// fall through on error
}
// last resort
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
static bool win_is_out_of_memory_error(DWORD err) {
switch (err) {
case ERROR_COMMITMENT_MINIMUM:
case ERROR_COMMITMENT_LIMIT:
case ERROR_PAGEFILE_QUOTA:
case ERROR_NOT_ENOUGH_MEMORY:
return true;
default:
return false;
}
}
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds
if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to "true"
for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms)
void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);
if (p != NULL) {
// success, return the address
return p;
}
else if (max_retry_msecs > 0 && (try_alignment <= 2*MI_SEGMENT_ALIGN) &&
(flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&
win_is_out_of_memory_error(GetLastError())) {
// if committing regular memory and being out-of-memory,
// keep trying for a bit in case memory frees up after all. See issue #894
_mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags);
long sleep_msecs = tries*40; // increasing waits
if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }
max_retry_msecs -= sleep_msecs;
Sleep(sleep_msecs);
}
else {
// otherwise return with an error
break;
}
}
return NULL;
}
static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
mi_assert_internal(!(large_only && !allow_large));
static _Atomic(size_t) large_page_try_ok; // = 0;
@ -572,6 +612,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
#if !defined(MI_SHARED_LIB)
// use thread local storage keys to detect thread ending
// note: another design could be to use special linker sections (see issue #869)
#include <fibersapi.h>
#if (_WIN32_WINNT < 0x600) // before Windows Vista
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );

34
src/segment.c
View File

@ -347,7 +347,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
}
static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, size_t* info_slices) {
static size_t mi_segment_calculate_slices(size_t required, size_t* info_slices) {
size_t page_size = _mi_os_page_size();
size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size);
size_t guardsize = 0;
@ -361,7 +361,6 @@ static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, siz
}
}
if (pre_size != NULL) *pre_size = isize;
isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE);
if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE;
size_t segment_size = (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + guardsize, MI_SEGMENT_SLICE_SIZE) );
@ -624,7 +623,9 @@ static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size
mi_assert_internal(slice->slice_count == slice_count); // no overflow?
slice->slice_offset = 0;
if (slice_count > 1) {
mi_slice_t* last = &segment->slices[slice_index + slice_count - 1];
mi_slice_t* last = slice + slice_count - 1;
mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment);
if (last > end) { last = end; }
last->slice_count = 0;
last->slice_offset = (uint32_t)(sizeof(mi_page_t)*(slice_count - 1));
last->block_size = 0;
@ -808,7 +809,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren
----------------------------------------------------------- */
static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id,
size_t* psegment_slices, size_t* ppre_size, size_t* pinfo_slices,
size_t* psegment_slices, size_t* pinfo_slices,
bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{
@ -825,7 +826,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment
align_offset = _mi_align_up( info_size, MI_SEGMENT_ALIGN );
const size_t extra = align_offset - info_size;
// recalculate due to potential guard pages
*psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices);
*psegment_slices = mi_segment_calculate_slices(required + extra, pinfo_slices);
mi_assert_internal(*psegment_slices > 0 && *psegment_slices <= UINT32_MAX);
}
@ -874,8 +875,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
// calculate needed sizes first
size_t info_slices;
size_t pre_size;
size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices);
size_t segment_slices = mi_segment_calculate_slices(required, &info_slices);
mi_assert_internal(segment_slices > 0 && segment_slices <= UINT32_MAX);
// Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little)
@ -887,7 +887,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
// Allocate the segment from the OS
mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id,
&segment_slices, &pre_size, &info_slices, commit, tld, os_tld);
&segment_slices, &info_slices, commit, tld, os_tld);
if (segment == NULL) return NULL;
// zero the segment info? -- not always needed as it may be zero initialized from the OS
@ -916,7 +916,6 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
// in secure mode, we set up a protected page in between the segment info
// and the page data, and at the end of the segment.
size_t os_pagesize = _mi_os_page_size();
mi_assert_internal(mi_segment_info_size(segment) - os_pagesize >= pre_size);
_mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize);
uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize;
mi_segment_ensure_committed(segment, end, os_pagesize, tld->stats);
@ -1007,11 +1006,13 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld
_mi_os_reset(start, psize, tld->stats);
}
// zero the page data, but not the segment fields
// zero the page data, but not the segment fields and heap tag
page->is_zero_init = false;
uint8_t heap_tag = page->heap_tag;
ptrdiff_t ofs = offsetof(mi_page_t, capacity);
_mi_memzero((uint8_t*)page + ofs, sizeof(*page) - ofs);
page->block_size = 1;
page->heap_tag = heap_tag;
// and free it
mi_slice_t* slice = mi_segment_span_free_coalesce(mi_page_to_slice(page), tld);
@ -1212,8 +1213,13 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap,
mi_assert_internal(page->next == NULL && page->prev==NULL);
_mi_stat_decrease(&tld->stats->pages_abandoned, 1);
segment->abandoned--;
// set the heap again and allow delayed free again
mi_page_set_heap(page, heap);
// set the heap again and allow heap thread delayed free again.
mi_heap_t* target_heap = _mi_heap_by_tag(heap, page->heap_tag); // allow custom heaps to separate objects
if (target_heap == NULL) {
target_heap = heap;
_mi_error_message(EINVAL, "page with tag %u cannot be reclaimed by a heap with the same tag (using %u instead)\n", page->heap_tag, heap->tag );
}
mi_page_set_heap(page, target_heap);
_mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set)
_mi_page_free_collect(page, false); // ensure used count is up to date
if (mi_page_all_free(page)) {
@ -1222,8 +1228,8 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap,
}
else {
// otherwise reclaim it into the heap
_mi_page_reclaim(heap, page);
if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page)) {
_mi_page_reclaim(target_heap, page);
if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page) && heap == target_heap) {
if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; }
}
}