merge dev branch
This commit is contained in:
daan
commit
663769c512
@ -89,7 +89,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
#define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE)
|
||||
#define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE)
|
||||
|
||||
#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1MiB on 64-bit
|
||||
#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/8) // 512KiB on 64-bit
|
||||
#define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT)
|
||||
|
||||
|
||||
|
||||
@ -377,6 +377,7 @@ typedef enum patch_apply_e {
|
||||
|
||||
typedef struct mi_patch_s {
|
||||
const char* name; // name of the function to patch
|
||||
int priority; // priority to patch this one (used to prioritize over multiple entries in various dll's)
|
||||
void* original; // the resolved address of the function (or NULL)
|
||||
void* target; // the address of the new target (never NULL)
|
||||
void* target_term;// the address of the target during termination (or NULL)
|
||||
@ -384,8 +385,8 @@ typedef struct mi_patch_s {
|
||||
mi_jump_t save; // the saved instructions in case it was applied
|
||||
} mi_patch_t;
|
||||
|
||||
#define MI_PATCH_NAME3(name,target,term) { name, NULL, &target, &term, false }
|
||||
#define MI_PATCH_NAME2(name,target) { name, NULL, &target, NULL, false }
|
||||
#define MI_PATCH_NAME3(name,target,term) { name, 0, NULL, &target, &term, PATCH_NONE }
|
||||
#define MI_PATCH_NAME2(name,target) { name, 0, NULL, &target, NULL, PATCH_NONE }
|
||||
#define MI_PATCH3(name,target,term) MI_PATCH_NAME3(#name, target, term)
|
||||
#define MI_PATCH2(name,target) MI_PATCH_NAME2(#name, target)
|
||||
#define MI_PATCH1(name) MI_PATCH2(name,mi_##name)
|
||||
@ -447,7 +448,7 @@ static mi_patch_t patches[] = {
|
||||
MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free, mi_free_term),
|
||||
#endif
|
||||
|
||||
{ NULL, NULL, NULL, false }
|
||||
{ NULL, 0, NULL, NULL, NULL, PATCH_NONE }
|
||||
};
|
||||
|
||||
|
||||
@ -522,15 +523,16 @@ static int __cdecl mi_setmaxstdio(int newmax) {
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Try to resolve patches for a given module (DLL)
|
||||
static void mi_module_resolve(HMODULE mod) {
|
||||
static void mi_module_resolve(HMODULE mod, int priority) {
|
||||
// see if any patches apply
|
||||
for (size_t i = 0; patches[i].name != NULL; i++) {
|
||||
mi_patch_t* patch = &patches[i];
|
||||
if (!patch->applied && patch->original==NULL) {
|
||||
if (!patch->applied && patch->priority < priority) {
|
||||
void* addr = GetProcAddress(mod, patch->name);
|
||||
if (addr != NULL) {
|
||||
// found it! set the address
|
||||
patch->original = addr;
|
||||
patch->priority = priority;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -554,9 +556,11 @@ static bool mi_patches_resolve(void) {
|
||||
size_t count = needed / sizeof(HMODULE);
|
||||
size_t ucrtbase_index = 0;
|
||||
size_t mimalloc_index = 0;
|
||||
// iterate through the loaded modules
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
HMODULE mod = modules[i];
|
||||
// iterate through the loaded modules; do this from the end so we prefer the
|
||||
// first loaded DLL as sometimes both "msvcr" and "ucrt" are both loaded and we should
|
||||
// override "ucrt" in that situation.
|
||||
for (size_t i = count; i > 0; i--) {
|
||||
HMODULE mod = modules[i-1];
|
||||
char filename[MAX_PATH] = { 0 };
|
||||
DWORD slen = GetModuleFileName(mod, filename, MAX_PATH);
|
||||
if (slen > 0 && slen < MAX_PATH) {
|
||||
@ -564,16 +568,19 @@ static bool mi_patches_resolve(void) {
|
||||
filename[slen] = 0;
|
||||
const char* lastsep = strrchr(filename, '\\');
|
||||
const char* basename = (lastsep==NULL ? filename : lastsep+1);
|
||||
if (i==0 // main module to allow static crt linking
|
||||
|| _strnicmp(basename, "ucrt", 4) == 0 // new ucrtbase.dll in windows 10
|
||||
|| _strnicmp(basename, "msvcr", 5) == 0) // older runtimes
|
||||
{
|
||||
|
||||
int priority = 0;
|
||||
if (i == 0) priority = 2; // main module to allow static crt linking
|
||||
else if (_strnicmp(basename, "ucrt", 4) == 0) priority = 3; // new ucrtbase.dll in windows 10
|
||||
else if (_strnicmp(basename, "msvcr", 5) == 0) priority = 1; // older runtimes
|
||||
|
||||
if (priority > 0) {
|
||||
// remember indices so we can check load order (in debug mode)
|
||||
if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i;
|
||||
if (_stricmp(basename, UCRTBASE_NAME) == 0) ucrtbase_index = i;
|
||||
|
||||
// probably found a crt module, try to patch it
|
||||
mi_module_resolve(mod);
|
||||
mi_module_resolve(mod,priority);
|
||||
|
||||
// try to find the atexit functions for the main process (in `ucrtbase.dll`)
|
||||
if (crt_atexit==NULL) crt_atexit = (atexit_fun_t*)GetProcAddress(mod, "_crt_atexit");
|
||||
|
||||
@ -60,8 +60,8 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
void* calloc(size_t size, size_t n) mi_attr_noexcept MI_FORWARD2(mi_calloc, size, n);
|
||||
void* realloc(void* p, size_t newsize) mi_attr_noexcept MI_FORWARD2(mi_realloc, p, newsize);
|
||||
void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p);
|
||||
char* strdup(const char* s) MI_FORWARD1(mi_strdup, s);
|
||||
char* strndup(const char* s, size_t n) MI_FORWARD2(mi_strndup, s, n);
|
||||
//char* strdup(const char* s) MI_FORWARD1(mi_strdup, s);
|
||||
//char* strndup(const char* s, size_t n) MI_FORWARD2(mi_strndup, s, n);
|
||||
#endif
|
||||
|
||||
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
|
||||
|
||||
@ -103,7 +103,11 @@ mi_heap_t _mi_heap_main = {
|
||||
NULL,
|
||||
0,
|
||||
0,
|
||||
0xCDCDCDCDCDCDCDL,
|
||||
#if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!)
|
||||
0xCDCDCDCDCDCDCDCDUL,
|
||||
#else
|
||||
0xCDCDCDCDUL,
|
||||
#endif
|
||||
0,
|
||||
false // can reclaim
|
||||
};
|
||||
|
||||
462
src/os.c
462
src/os.c
@ -14,18 +14,38 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
#include <string.h> // memset
|
||||
#include <errno.h>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h> // mmap
|
||||
#include <unistd.h> // sysconf
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Initialization.
|
||||
On windows initializes support for aligned allocation and
|
||||
large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
|
||||
----------------------------------------------------------- */
|
||||
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h> // mmap
|
||||
#include <unistd.h> // sysconf
|
||||
#endif
|
||||
uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
|
||||
uintptr_t x = (sz / alignment) * alignment;
|
||||
if (x < sz) x += alignment;
|
||||
if (x < sz) return 0; // overflow
|
||||
return x;
|
||||
}
|
||||
|
||||
static void* mi_align_up_ptr(void* p, size_t alignment) {
|
||||
return (void*)_mi_align_up((uintptr_t)p, alignment);
|
||||
}
|
||||
|
||||
static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
|
||||
return (sz / alignment) * alignment;
|
||||
}
|
||||
|
||||
static void* mi_align_down_ptr(void* p, size_t alignment) {
|
||||
return (void*)_mi_align_down((uintptr_t)p, alignment);
|
||||
}
|
||||
|
||||
// page size (initialized properly in `os_init`)
|
||||
static size_t os_page_size = 4096;
|
||||
@ -61,8 +81,8 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
|
||||
|
||||
#if defined(_WIN32)
|
||||
// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
|
||||
// So, we need to look it up dynamically to run on older systems.
|
||||
typedef PVOID (*VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG );
|
||||
// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
|
||||
typedef PVOID(__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
|
||||
static VirtualAlloc2Ptr pVirtualAlloc2 = NULL;
|
||||
|
||||
void _mi_os_init(void) {
|
||||
@ -74,7 +94,7 @@ void _mi_os_init(void) {
|
||||
// get the VirtualAlloc2 function
|
||||
HINSTANCE hDll;
|
||||
hDll = LoadLibrary("kernelbase.dll");
|
||||
if (hDll!=NULL) {
|
||||
if (hDll != NULL) {
|
||||
// use VirtualAlloc2FromApp as it is available to Windows store apps
|
||||
pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp");
|
||||
FreeLibrary(hDll);
|
||||
@ -106,7 +126,7 @@ void _mi_os_init(void) {
|
||||
CloseHandle(token);
|
||||
}
|
||||
if (!ok) {
|
||||
if (err==0) err = GetLastError();
|
||||
if (err == 0) err = GetLastError();
|
||||
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
|
||||
}
|
||||
}
|
||||
@ -120,7 +140,7 @@ void _mi_os_init() {
|
||||
os_alloc_granularity = os_page_size;
|
||||
}
|
||||
if (mi_option_is_enabled(mi_option_large_os_pages)) {
|
||||
large_os_page_size = (1UL<<21); // 2MiB
|
||||
large_os_page_size = (1UL << 21); // 2MiB
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -128,29 +148,8 @@ void _mi_os_init() {
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).
|
||||
Defines a portable `mmap`, `munmap` and `mmap_trim`.
|
||||
----------------------------------------------------------- */
|
||||
|
||||
uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
|
||||
uintptr_t x = (sz / alignment) * alignment;
|
||||
if (x < sz) x += alignment;
|
||||
if (x < sz) return 0; // overflow
|
||||
return x;
|
||||
}
|
||||
|
||||
static void* mi_align_up_ptr(void* p, size_t alignment) {
|
||||
return (void*)_mi_align_up((uintptr_t)p, alignment);
|
||||
}
|
||||
|
||||
static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
|
||||
return (sz / alignment) * alignment;
|
||||
}
|
||||
|
||||
static void* mi_align_down_ptr(void* p, size_t alignment) {
|
||||
return (void*)_mi_align_down((uintptr_t)p, alignment);
|
||||
}
|
||||
|
||||
|
||||
static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
|
||||
{
|
||||
if (addr == NULL || size == 0) return true;
|
||||
@ -163,7 +162,7 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
|
||||
_mi_stat_decrease(&stats->committed, size); // TODO: what if never committed?
|
||||
_mi_stat_decrease(&stats->reserved, size);
|
||||
if (err) {
|
||||
#pragma warning(suppress:4996)
|
||||
#pragma warning(suppress:4996)
|
||||
_mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size);
|
||||
return false;
|
||||
}
|
||||
@ -172,39 +171,53 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
|
||||
}
|
||||
}
|
||||
|
||||
static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_flags, mi_stats_t* stats) {
|
||||
UNUSED(stats);
|
||||
if (size == 0) return NULL;
|
||||
#ifdef _WIN32
|
||||
static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
|
||||
if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
|
||||
// on modern Windows try use VirtualAlloc2
|
||||
MEM_ADDRESS_REQUIREMENTS reqs = { 0 };
|
||||
reqs.Alignment = try_alignment;
|
||||
MEM_EXTENDED_PARAMETER param = { 0 };
|
||||
param.Type = MemExtendedParameterAddressRequirements;
|
||||
param.Pointer = &reqs;
|
||||
return (*pVirtualAlloc2)(addr, NULL, size, flags, PAGE_READWRITE, ¶m, 1);
|
||||
}
|
||||
#endif
|
||||
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
|
||||
}
|
||||
|
||||
static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
void* p = NULL;
|
||||
#if defined(_WIN32)
|
||||
int flags = MEM_RESERVE | extra_flags;
|
||||
if (commit) flags |= MEM_COMMIT;
|
||||
if (use_large_os_page(size, 0)) {
|
||||
p = VirtualAlloc(addr, size, MEM_LARGE_PAGES | flags, PAGE_READWRITE);
|
||||
if (use_large_os_page(size, try_alignment)) {
|
||||
p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | flags);
|
||||
// fall back to non-large page allocation on error (`p == NULL`).
|
||||
}
|
||||
if (p == NULL) {
|
||||
p = VirtualAlloc(addr, size, flags, PAGE_READWRITE);
|
||||
p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
#else
|
||||
static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) {
|
||||
void* p = NULL;
|
||||
#if !defined(MAP_ANONYMOUS)
|
||||
#define MAP_ANONYMOUS MAP_ANON
|
||||
#endif
|
||||
int flags = MAP_PRIVATE | MAP_ANONYMOUS | extra_flags;
|
||||
if (addr != NULL) {
|
||||
#if defined(MAP_EXCL)
|
||||
flags |= MAP_FIXED | MAP_EXCL; // BSD
|
||||
#elif defined(MAP_FIXED_NOREPLACE)
|
||||
flags |= MAP_FIXED_NOREPLACE; // Linux
|
||||
#elif defined(MAP_FIXED)
|
||||
flags |= MAP_FIXED;
|
||||
#endif
|
||||
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
|
||||
#if defined(MAP_ALIGNED) // BSD
|
||||
if (try_alignment > 0) {
|
||||
size_t n = _mi_bsr(try_alignment);
|
||||
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
|
||||
flags |= MAP_ALIGNED(n);
|
||||
}
|
||||
}
|
||||
int pflags = (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE);
|
||||
#if defined(PROT_MAX)
|
||||
pflags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
|
||||
#endif
|
||||
|
||||
if (large_os_page_size > 0 && use_large_os_page(size, 0) && ((uintptr_t)addr % large_os_page_size) == 0) {
|
||||
#if defined(PROT_MAX)
|
||||
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
|
||||
#endif
|
||||
if (large_os_page_size > 0 && use_large_os_page(size, try_alignment)) {
|
||||
int lflags = flags;
|
||||
#ifdef MAP_ALIGNED_SUPER
|
||||
lflags |= MAP_ALIGNED_SUPER;
|
||||
@ -217,74 +230,155 @@ static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_fla
|
||||
#endif
|
||||
if (lflags != flags) {
|
||||
// try large page allocation
|
||||
p = mmap(addr, size, pflags, lflags, -1, 0);
|
||||
// TODO: if always failing due to permissions or no huge pages, try to avoid repeatedly trying?
|
||||
// Should we check this in _mi_os_init? (as on Windows)
|
||||
p = mmap(NULL, size, protect_flags, lflags, -1, 0);
|
||||
if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission
|
||||
}
|
||||
}
|
||||
if (p == NULL) {
|
||||
p = mmap(addr, size, pflags, flags, -1, 0);
|
||||
p = mmap(NULL, size, protect_flags, flags, -1, 0);
|
||||
if (p == MAP_FAILED) p = NULL;
|
||||
}
|
||||
if (addr != NULL && p != addr) {
|
||||
return p;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Primitive allocation from the OS.
|
||||
// Note: the `alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) {
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
if (size == 0) return NULL;
|
||||
|
||||
void* p = NULL;
|
||||
#if defined(_WIN32)
|
||||
int flags = MEM_RESERVE;
|
||||
if (commit) flags |= MEM_COMMIT;
|
||||
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags);
|
||||
#else
|
||||
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
|
||||
p = mi_unix_mmap(size, try_alignment, protect_flags);
|
||||
#endif
|
||||
_mi_stat_increase(&stats->mmap_calls, 1);
|
||||
if (p != NULL) {
|
||||
_mi_stat_increase(&stats->reserved, size);
|
||||
if (commit) _mi_stat_increase(&stats->committed, size);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
// Primitive aligned allocation from the OS.
|
||||
// This function guarantees the allocated memory is aligned.
|
||||
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) {
|
||||
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
|
||||
size = _mi_align_up(size, _mi_os_page_size());
|
||||
|
||||
// try first with a hint (this will be aligned directly on Win 10+ or BSD)
|
||||
void* p = mi_os_mem_alloc(size, alignment, commit, stats);
|
||||
if (p == NULL) return NULL;
|
||||
|
||||
// if not aligned, free it, overallocate, and unmap around it
|
||||
if (((uintptr_t)p % alignment != 0)) {
|
||||
mi_os_mem_free(p, size, stats);
|
||||
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
|
||||
size_t over_size = size + alignment;
|
||||
|
||||
#if _WIN32
|
||||
// over-allocate and than re-allocate exactly at an aligned address in there.
|
||||
// this may fail due to threads allocating at the same time so we
|
||||
// retry this at most 3 times before giving up.
|
||||
// (we can not decommit around the overallocation on Windows, because we can only
|
||||
// free the original pointer, not one pointing inside the area)
|
||||
int flags = MEM_RESERVE;
|
||||
if (commit) flags |= MEM_COMMIT;
|
||||
for (int tries = 0; tries < 3; tries++) {
|
||||
// over-allocate to determine a virtual memory range
|
||||
p = mi_os_mem_alloc(over_size, alignment, commit, stats);
|
||||
if (p == NULL) return NULL; // error
|
||||
if (((uintptr_t)p % alignment) == 0) {
|
||||
// if p happens to be aligned, just decommit the left-over area
|
||||
_mi_os_decommit((uint8_t*)p + size, over_size - size, stats);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
// otherwise free and allocate at an aligned address in there
|
||||
mi_os_mem_free(p, over_size, stats);
|
||||
void* aligned_p = mi_align_up_ptr(p, alignment);
|
||||
p = mi_win_virtual_alloc(aligned_p, size, alignment, flags);
|
||||
if (p == aligned_p) break; // success!
|
||||
if (p != NULL) { // should not happen?
|
||||
mi_os_mem_free(p, size, stats);
|
||||
p = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
// overallocate...
|
||||
p = mi_os_mem_alloc(over_size, alignment, commit, stats);
|
||||
if (p == NULL) return NULL;
|
||||
// and selectively unmap parts around the over-allocated area.
|
||||
void* aligned_p = mi_align_up_ptr(p, alignment);
|
||||
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
|
||||
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
|
||||
size_t post_size = over_size - pre_size - mid_size;
|
||||
mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
|
||||
if (pre_size > 0) mi_os_mem_free(p, pre_size, stats);
|
||||
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
|
||||
// we can return the aligned pointer on `mmap` systems
|
||||
p = aligned_p;
|
||||
#endif
|
||||
UNUSED(stats);
|
||||
mi_assert(p == NULL || (addr == NULL && p != addr) || (addr != NULL && p == addr));
|
||||
if (p != NULL) {
|
||||
mi_stat_increase(stats->mmap_calls, 1);
|
||||
mi_stat_increase(stats->reserved, size);
|
||||
if (commit) mi_stat_increase(stats->committed, size);
|
||||
}
|
||||
|
||||
mi_assert_internal(p == NULL || (p != NULL && ((uintptr_t)p % alignment) == 0));
|
||||
return p;
|
||||
}
|
||||
|
||||
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) {
|
||||
if (alignment < _mi_os_page_size() || ((alignment & (~alignment + 1)) != alignment)) return NULL;
|
||||
void* p = NULL;
|
||||
#if defined(_WIN32) && defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
|
||||
if (pVirtualAlloc2 != NULL) {
|
||||
// on modern Windows try use VirtualAlloc2
|
||||
MEM_ADDRESS_REQUIREMENTS reqs = {0};
|
||||
reqs.Alignment = alignment;
|
||||
MEM_EXTENDED_PARAMETER param = { 0 };
|
||||
param.Type = MemExtendedParameterAddressRequirements;
|
||||
param.Pointer = &reqs;
|
||||
DWORD flags = MEM_RESERVE;
|
||||
if (commit) flags |= MEM_COMMIT;
|
||||
if (use_large_os_page(size, alignment)) flags |= MEM_LARGE_PAGES;
|
||||
p = (*pVirtualAlloc2)(NULL, NULL, size, flags, PAGE_READWRITE, ¶m, 1);
|
||||
}
|
||||
#elif defined(MAP_ALIGNED)
|
||||
// on BSD, use the aligned mmap api
|
||||
size_t n = _mi_bsr(alignment);
|
||||
if (((size_t)1 << n) == alignment && n >= 12) { // alignment is a power of 2 and >= 4096
|
||||
p = mi_os_mem_alloc(suggest, size, commit, MAP_ALIGNED(n), tld->stats); // use the NetBSD/freeBSD aligned flags
|
||||
}
|
||||
#else
|
||||
UNUSED(size);
|
||||
UNUSED(alignment);
|
||||
#endif
|
||||
UNUSED(stats); // if !STATS
|
||||
mi_assert(p == NULL || (uintptr_t)p % alignment == 0);
|
||||
if (p != NULL) {
|
||||
mi_stat_increase(stats->mmap_calls, 1);
|
||||
mi_stat_increase(stats->reserved, size);
|
||||
if (commit) mi_stat_increase(stats->committed, size);
|
||||
}
|
||||
return p;
|
||||
/* -----------------------------------------------------------
|
||||
OS API: alloc, free, alloc_aligned
|
||||
----------------------------------------------------------- */
|
||||
|
||||
void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
|
||||
if (size == 0) return NULL;
|
||||
size = mi_os_good_alloc_size(size, 0);
|
||||
return mi_os_mem_alloc(size, 0, true, stats);
|
||||
}
|
||||
|
||||
// Conservatively OS page align within a given area
|
||||
static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) {
|
||||
void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
|
||||
if (size == 0 || p == NULL) return;
|
||||
size = mi_os_good_alloc_size(size, 0);
|
||||
mi_os_mem_free(p, size, stats);
|
||||
}
|
||||
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
|
||||
{
|
||||
if (size == 0) return NULL;
|
||||
size = mi_os_good_alloc_size(size, alignment);
|
||||
alignment = _mi_align_up(alignment, _mi_os_page_size());
|
||||
return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
OS memory API: reset, commit, decommit, protect, unprotect.
|
||||
----------------------------------------------------------- */
|
||||
|
||||
|
||||
// OS page align within a given area, either conservative (pages inside the area only),
|
||||
// or not (straddling pages outside the area is possible)
|
||||
static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
|
||||
mi_assert(addr != NULL && size > 0);
|
||||
if (newsize != NULL) *newsize = 0;
|
||||
if (size == 0 || addr == NULL) return NULL;
|
||||
|
||||
// page align conservatively within the range
|
||||
void* start = mi_align_up_ptr(addr, _mi_os_page_size());
|
||||
void* end = mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size());
|
||||
void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
|
||||
: mi_align_down_ptr(addr, _mi_os_page_size()));
|
||||
void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
|
||||
: mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
|
||||
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
|
||||
if (diff <= 0) return NULL;
|
||||
|
||||
@ -293,6 +387,12 @@ static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) {
|
||||
return start;
|
||||
}
|
||||
|
||||
static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
|
||||
return mi_os_page_align_areax(true, addr, size, newsize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Signal to the OS that the address range is no longer in use
|
||||
// but may be used later again. This will release physical memory
|
||||
// pages and reduce swapping while keeping the memory committed.
|
||||
@ -300,10 +400,9 @@ static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) {
|
||||
bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
|
||||
// page align conservatively within the range
|
||||
size_t csize;
|
||||
void* start = mi_os_page_align_area(addr,size,&csize);
|
||||
if (csize==0) return true;
|
||||
UNUSED(stats); // if !STATS
|
||||
mi_stat_increase(stats->reset, csize);
|
||||
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
|
||||
if (csize == 0) return true;
|
||||
_mi_stat_increase(&stats->reset, csize);
|
||||
|
||||
#if defined(_WIN32)
|
||||
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
|
||||
@ -323,17 +422,17 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
|
||||
*/
|
||||
return true;
|
||||
#else
|
||||
#if defined(MADV_FREE)
|
||||
#if defined(MADV_FREE)
|
||||
static int advice = MADV_FREE;
|
||||
int err = madvise(start, csize, advice);
|
||||
if (err!=0 && errno==EINVAL && advice==MADV_FREE) {
|
||||
if (err != 0 && errno == EINVAL && advice == MADV_FREE) {
|
||||
// if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
|
||||
advice = MADV_DONTNEED;
|
||||
err = madvise(start, csize, advice);
|
||||
}
|
||||
#else
|
||||
#else
|
||||
int err = madvise(start, csize, MADV_DONTNEED);
|
||||
#endif
|
||||
#endif
|
||||
if (err != 0) {
|
||||
_mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno);
|
||||
}
|
||||
@ -346,46 +445,44 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
|
||||
static bool mi_os_protectx(void* addr, size_t size, bool protect) {
|
||||
// page align conservatively within the range
|
||||
size_t csize = 0;
|
||||
void* start = mi_os_page_align_area(addr, size, &csize);
|
||||
if (csize==0) return false;
|
||||
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
|
||||
if (csize == 0) return false;
|
||||
|
||||
int err = 0;
|
||||
#ifdef _WIN32
|
||||
DWORD oldprotect = 0;
|
||||
BOOL ok = VirtualProtect(start,csize,protect ? PAGE_NOACCESS : PAGE_READWRITE,&oldprotect);
|
||||
BOOL ok = VirtualProtect(start, csize, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);
|
||||
err = (ok ? 0 : GetLastError());
|
||||
#else
|
||||
err = mprotect(start,csize,protect ? PROT_NONE : (PROT_READ|PROT_WRITE));
|
||||
err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
|
||||
#endif
|
||||
if (err != 0) {
|
||||
_mi_warning_message("mprotect error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err);
|
||||
}
|
||||
return (err==0);
|
||||
return (err == 0);
|
||||
}
|
||||
|
||||
bool _mi_os_protect(void* addr, size_t size) {
|
||||
return mi_os_protectx(addr,size,true);
|
||||
return mi_os_protectx(addr, size, true);
|
||||
}
|
||||
|
||||
bool _mi_os_unprotect(void* addr, size_t size) {
|
||||
return mi_os_protectx(addr, size, false);
|
||||
}
|
||||
|
||||
// Commit/Decommit memory.
|
||||
// We page align to a conservative area inside the range to reset.
|
||||
// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative.
|
||||
static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) {
|
||||
// page align conservatively within the range
|
||||
// page align in the range, commit liberally, decommit conservative
|
||||
size_t csize;
|
||||
void* start = mi_os_page_align_area(addr, size, &csize);
|
||||
void* start = mi_os_page_align_areax(!commit, addr, size, &csize);
|
||||
if (csize == 0) return true;
|
||||
int err = 0;
|
||||
UNUSED(stats); // if !STATS
|
||||
if (commit) {
|
||||
mi_stat_increase(stats->committed, csize);
|
||||
mi_stat_increase(stats->commit_calls,1);
|
||||
_mi_stat_increase(&stats->committed, csize);
|
||||
_mi_stat_increase(&stats->commit_calls, 1);
|
||||
}
|
||||
else {
|
||||
mi_stat_decrease(stats->committed, csize);
|
||||
_mi_stat_decrease(&stats->committed, csize);
|
||||
}
|
||||
|
||||
#if defined(_WIN32)
|
||||
@ -415,107 +512,22 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
|
||||
return mi_os_commitx(addr, size, false, stats);
|
||||
}
|
||||
|
||||
bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
|
||||
// page align conservatively within the range
|
||||
mi_assert_internal(oldsize > newsize && p != NULL);
|
||||
if (oldsize < newsize || p == NULL) return false;
|
||||
if (oldsize == newsize) return true;
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
OS allocation using mmap/munmap
|
||||
----------------------------------------------------------- */
|
||||
// oldsize and newsize should be page aligned or we cannot shrink precisely
|
||||
void* addr = (uint8_t*)p + newsize;
|
||||
size_t size = 0;
|
||||
void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size);
|
||||
if (size == 0 || start != addr) return false;
|
||||
|
||||
void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
|
||||
if (size == 0) return NULL;
|
||||
size = mi_os_good_alloc_size(size, 0);
|
||||
void* p = mi_os_mem_alloc(NULL, size, true, 0, stats);
|
||||
mi_assert(p!=NULL);
|
||||
return p;
|
||||
}
|
||||
|
||||
void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
|
||||
UNUSED(stats);
|
||||
mi_os_mem_free(p, size, stats);
|
||||
}
|
||||
|
||||
// Slow but guaranteed way to allocated aligned memory
|
||||
// by over-allocating and then reallocating at a fixed aligned
|
||||
// address that should be available then.
|
||||
static void* mi_os_alloc_aligned_ensured(size_t size, size_t alignment, bool commit, size_t trie, mi_stats_t* stats)
|
||||
{
|
||||
if (trie >= 3) return NULL; // stop recursion (only on Windows)
|
||||
size_t alloc_size = size + alignment;
|
||||
mi_assert(alloc_size >= size); // overflow?
|
||||
if (alloc_size < size) return NULL;
|
||||
|
||||
// allocate a chunk that includes the alignment
|
||||
void* p = mi_os_mem_alloc(NULL, alloc_size, commit, 0, stats);
|
||||
if (p == NULL) return NULL;
|
||||
// create an aligned pointer in the allocated area
|
||||
void* aligned_p = mi_align_up_ptr(p, alignment);
|
||||
mi_assert(aligned_p != NULL);
|
||||
|
||||
// free it and try to allocate `size` at exactly `aligned_p`
|
||||
// note: this may fail in case another thread happens to allocate
|
||||
// concurrently at that spot. We try up to 3 times to mitigate this.
|
||||
mi_os_mem_free(p, alloc_size, stats);
|
||||
p = mi_os_mem_alloc(aligned_p, size, commit, 0, stats);
|
||||
if (p != aligned_p) {
|
||||
if (p != NULL) mi_os_mem_free(p, size, stats);
|
||||
return mi_os_alloc_aligned_ensured(size, alignment, commit, trie++, stats);
|
||||
}
|
||||
#if 0 // could use this on mmap systems
|
||||
// we selectively unmap parts around the over-allocated area.
|
||||
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
|
||||
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
|
||||
size_t post_size = alloc_size - pre_size - mid_size;
|
||||
if (pre_size > 0) mi_os_mem_free(p, pre_size, stats);
|
||||
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
|
||||
#ifdef _WIN32
|
||||
// we cannot shrink on windows, but we can decommit
|
||||
return _mi_os_decommit(start, size, stats);
|
||||
#else
|
||||
return mi_os_mem_free(start, size, stats);
|
||||
#endif
|
||||
|
||||
mi_assert(((uintptr_t)aligned_p) % alignment == 0);
|
||||
return aligned_p;
|
||||
}
|
||||
|
||||
// Allocate an aligned block.
|
||||
// Since `mi_mmap` is relatively slow we try to allocate directly at first and
|
||||
// hope to get an aligned address; only when that fails we fall back
|
||||
// to a guaranteed method by overallocating at first and adjusting.
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
|
||||
{
|
||||
if (size == 0) return NULL;
|
||||
size = mi_os_good_alloc_size(size,alignment);
|
||||
if (alignment < 1024) return mi_os_mem_alloc(NULL, size, commit, 0, tld->stats);
|
||||
|
||||
// try direct OS aligned allocation; only supported on BSD and Windows 10+
|
||||
void* suggest = NULL;
|
||||
void* p = mi_os_mem_alloc_aligned(size,alignment,commit,tld->stats);
|
||||
|
||||
// Fall back
|
||||
if (p==NULL && (tld->mmap_next_probable % alignment) == 0) {
|
||||
// if the next probable address is aligned,
|
||||
// then try to just allocate `size` and hope it is aligned...
|
||||
p = mi_os_mem_alloc(suggest, size, commit, 0, tld->stats);
|
||||
if (p == NULL) return NULL;
|
||||
if (((uintptr_t)p % alignment) == 0) mi_stat_increase(tld->stats->mmap_right_align, 1);
|
||||
}
|
||||
//fprintf(stderr, "segment address guess: %s, p=%lxu, guess:%lxu\n", (p != NULL && (uintptr_t)p % alignment ==0 ? "correct" : "incorrect"), (uintptr_t)p, next_probable);
|
||||
|
||||
if (p==NULL || ((uintptr_t)p % alignment) != 0) {
|
||||
// if `p` is not yet aligned after all, free the block and use a slower
|
||||
// but guaranteed way to allocate an aligned block
|
||||
if (p != NULL) mi_os_mem_free(p, size, tld->stats);
|
||||
mi_stat_increase( tld->stats->mmap_ensure_aligned, 1);
|
||||
//fprintf(stderr, "mimalloc: slow mmap 0x%lx\n", _mi_thread_id());
|
||||
p = mi_os_alloc_aligned_ensured(size, alignment,commit,0,tld->stats);
|
||||
}
|
||||
if (p != NULL) {
|
||||
// next probable address is the page-aligned address just after the newly allocated area.
|
||||
size_t probable_size = MI_SEGMENT_SIZE;
|
||||
if (tld->mmap_previous > p) {
|
||||
// Linux tends to allocate downward
|
||||
tld->mmap_next_probable = _mi_align_down((uintptr_t)p - probable_size, os_alloc_granularity); // ((uintptr_t)previous - (uintptr_t)p);
|
||||
}
|
||||
else {
|
||||
// Otherwise, guess the next address is page aligned `size` from current pointer
|
||||
tld->mmap_next_probable = _mi_align_up((uintptr_t)p + probable_size, os_alloc_granularity);
|
||||
}
|
||||
tld->mmap_previous = p;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
@ -391,6 +391,9 @@ void _mi_page_retire(mi_page_t* page) {
|
||||
static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats)
|
||||
{
|
||||
UNUSED(stats);
|
||||
mi_assert_internal(page->free == NULL);
|
||||
mi_assert_internal(page->local_free == NULL);
|
||||
mi_assert_internal(page->capacity + extend <= page->reserved);
|
||||
void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL );
|
||||
size_t bsize = page->block_size;
|
||||
mi_block_t* start = mi_page_block_at(page, page_area, page->capacity);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user