mirror of https://github.com/microsoft/mimalloc
initial commit of new primitive layer
This commit is contained in:
Daan Leijen
parent
2e6ab0f230
commit
08a01d26dc
|
|
@ -50,7 +50,8 @@ set(mi_sources
|
|||
src/alloc-posix.c
|
||||
src/heap.c
|
||||
src/options.c
|
||||
src/init.c)
|
||||
src/init.c
|
||||
src/prim/prim.c)
|
||||
|
||||
set(mi_cflags "")
|
||||
set(mi_libraries "")
|
||||
|
|
|
|||
|
|
@ -237,6 +237,7 @@
|
|||
<ClCompile Include="..\..\src\bitmap.c" />
|
||||
<ClCompile Include="..\..\src\heap.c" />
|
||||
<ClCompile Include="..\..\src\init.c" />
|
||||
<ClCompile Include="..\..\src\prim\prim.c" />
|
||||
<ClCompile Include="..\..\src\region.c" />
|
||||
<ClCompile Include="..\..\src\options.c" />
|
||||
<ClCompile Include="..\..\src\os.c" />
|
||||
|
|
|
|||
|
|
@ -225,6 +225,7 @@
|
|||
</ClCompile>
|
||||
<ClCompile Include="..\..\src\heap.c" />
|
||||
<ClCompile Include="..\..\src\init.c" />
|
||||
<ClCompile Include="..\..\src\prim\prim.c" />
|
||||
<ClCompile Include="..\..\src\region.c" />
|
||||
<ClCompile Include="..\..\src\options.c" />
|
||||
<ClCompile Include="..\..\src\page-queue.c">
|
||||
|
|
|
|||
|
|
@ -86,7 +86,9 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats);
|
|||
|
||||
void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool* large, mi_stats_t* tld_stats);
|
||||
void _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats);
|
||||
|
||||
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
|
||||
bool _mi_os_use_large_page(size_t size, size_t alignment);
|
||||
size_t _mi_os_large_page_size(void);
|
||||
|
||||
// memory.c
|
||||
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,483 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#ifndef _DEFAULT_SOURCE
|
||||
#define _DEFAULT_SOURCE // ensure mmap flags are defined
|
||||
#endif
|
||||
|
||||
#if defined(__sun)
|
||||
// illumos provides new mman.h api when any of these are defined
|
||||
// otherwise the old api based on caddr_t which predates the void pointers one.
|
||||
// stock solaris provides only the former, chose to atomically to discard those
|
||||
// flags only here rather than project wide tough.
|
||||
#undef _XOPEN_SOURCE
|
||||
#undef _POSIX_C_SOURCE
|
||||
#endif
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc-internal.h"
|
||||
#include "mimalloc-atomic.h"
|
||||
#include "prim.h"
|
||||
|
||||
#include <sys/mman.h> // mmap
|
||||
#include <unistd.h> // sysconf
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <features.h>
|
||||
#include <fcntl.h>
|
||||
#if defined(__GLIBC__)
|
||||
#include <linux/mman.h> // linux mmap flags
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#elif defined(__APPLE__)
|
||||
#include <TargetConditionals.h>
|
||||
#if !TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR
|
||||
#include <mach/vm_statistics.h>
|
||||
#endif
|
||||
#elif defined(__FreeBSD__) || defined(__DragonFly__)
|
||||
#include <sys/param.h>
|
||||
#if __FreeBSD_version >= 1200000
|
||||
#include <sys/cpuset.h>
|
||||
#include <sys/domainset.h>
|
||||
#endif
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
//---------------------------------------------
|
||||
// init
|
||||
//---------------------------------------------
|
||||
|
||||
static bool unix_detect_overcommit(void) {
|
||||
bool os_overcommit = true;
|
||||
#if defined(__linux__)
|
||||
int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
if (fd >= 0) {
|
||||
char buf[32];
|
||||
ssize_t nread = read(fd, &buf, sizeof(buf));
|
||||
close(fd);
|
||||
// <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
|
||||
// 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)
|
||||
if (nread >= 1) {
|
||||
os_overcommit = (buf[0] == '0' || buf[0] == '1');
|
||||
}
|
||||
}
|
||||
#elif defined(__FreeBSD__)
|
||||
int val = 0;
|
||||
size_t olen = sizeof(val);
|
||||
if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) {
|
||||
os_overcommit = (val != 0);
|
||||
}
|
||||
#else
|
||||
// default: overcommit is true
|
||||
#endif
|
||||
return os_overcommit;
|
||||
}
|
||||
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
|
||||
long psize = sysconf(_SC_PAGESIZE);
|
||||
if (psize > 0) {
|
||||
config->page_size = (size_t)psize;
|
||||
config->alloc_granularity = (size_t)psize;
|
||||
}
|
||||
config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this?
|
||||
config->has_overcommit = unix_detect_overcommit();
|
||||
config->must_free_whole = false; // mmap can free in parts
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// free
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_free(void* addr, size_t size ) {
|
||||
bool err = (munmap(addr, size) == -1);
|
||||
if (err) {
|
||||
_mi_warning_message("unable to release OS memory: %s, addr: %p, size: %zu\n", strerror(errno), addr, size);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// mmap
|
||||
//---------------------------------------------
|
||||
|
||||
static int unix_madvise(void* addr, size_t size, int advice) {
|
||||
#if defined(__sun)
|
||||
return madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)
|
||||
#else
|
||||
return madvise(addr, size, advice);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
|
||||
MI_UNUSED(try_alignment);
|
||||
#if defined(MAP_ALIGNED) // BSD
|
||||
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
size_t n = mi_bsr(try_alignment);
|
||||
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
|
||||
flags |= MAP_ALIGNED(n);
|
||||
void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0);
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
}
|
||||
#elif defined(MAP_ALIGN) // Solaris
|
||||
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
#endif
|
||||
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
|
||||
// on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
|
||||
if (addr == NULL) {
|
||||
void* hint = _mi_os_get_aligned_hint(try_alignment, size);
|
||||
if (hint != NULL) {
|
||||
void* p = mmap(hint, size, protect_flags, flags, fd, 0);
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// regular mmap
|
||||
void* p = mmap(addr, size, protect_flags, flags, fd, 0);
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// failed to allocate
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
|
||||
void* p = NULL;
|
||||
#if !defined(MAP_ANONYMOUS)
|
||||
#define MAP_ANONYMOUS MAP_ANON
|
||||
#endif
|
||||
#if !defined(MAP_NORESERVE)
|
||||
#define MAP_NORESERVE 0
|
||||
#endif
|
||||
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
|
||||
int fd = -1;
|
||||
if (_mi_os_has_overcommit()) {
|
||||
flags |= MAP_NORESERVE;
|
||||
}
|
||||
#if defined(PROT_MAX)
|
||||
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
|
||||
#endif
|
||||
#if defined(VM_MAKE_TAG)
|
||||
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
|
||||
int os_tag = (int)mi_option_get(mi_option_os_tag);
|
||||
if (os_tag < 100 || os_tag > 255) { os_tag = 100; }
|
||||
fd = VM_MAKE_TAG(os_tag);
|
||||
#endif
|
||||
// huge page allocation
|
||||
if ((large_only || _mi_os_use_large_page(size, try_alignment)) && allow_large) {
|
||||
static _Atomic(size_t) large_page_try_ok; // = 0;
|
||||
size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
|
||||
if (!large_only && try_ok > 0) {
|
||||
// If the OS is not configured for large OS pages, or the user does not have
|
||||
// enough permission, the `mmap` will always fail (but it might also fail for other reasons).
|
||||
// Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times
|
||||
// to avoid too many failing calls to mmap.
|
||||
mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
|
||||
}
|
||||
else {
|
||||
int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux
|
||||
int lfd = fd;
|
||||
#ifdef MAP_ALIGNED_SUPER
|
||||
lflags |= MAP_ALIGNED_SUPER;
|
||||
#endif
|
||||
#ifdef MAP_HUGETLB
|
||||
lflags |= MAP_HUGETLB;
|
||||
#endif
|
||||
#ifdef MAP_HUGE_1GB
|
||||
static bool mi_huge_pages_available = true;
|
||||
if ((size % MI_GiB) == 0 && mi_huge_pages_available) {
|
||||
lflags |= MAP_HUGE_1GB;
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
#ifdef MAP_HUGE_2MB
|
||||
lflags |= MAP_HUGE_2MB;
|
||||
#endif
|
||||
}
|
||||
#ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
|
||||
lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
|
||||
#endif
|
||||
if (large_only || lflags != flags) {
|
||||
// try large OS page allocation
|
||||
*is_large = true;
|
||||
p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd);
|
||||
#ifdef MAP_HUGE_1GB
|
||||
if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) {
|
||||
mi_huge_pages_available = false; // don't try huge 1GiB pages again
|
||||
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno);
|
||||
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
|
||||
p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd);
|
||||
}
|
||||
#endif
|
||||
if (large_only) return p;
|
||||
if (p == NULL) {
|
||||
mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// regular allocation
|
||||
if (p == NULL) {
|
||||
*is_large = false;
|
||||
p = unix_mmap_prim(addr, size, try_alignment, protect_flags, flags, fd);
|
||||
if (p != NULL) {
|
||||
#if defined(MADV_HUGEPAGE)
|
||||
// Many Linux systems don't allow MAP_HUGETLB but they support instead
|
||||
// transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
|
||||
// though since properly aligned allocations will already use large pages if available
|
||||
// in that case -- in particular for our large regions (in `memory.c`).
|
||||
// However, some systems only allow THP if called with explicit `madvise`, so
|
||||
// when large OS pages are enabled for mimalloc, we call `madvise` anyways.
|
||||
if (allow_large && _mi_os_use_large_page(size, try_alignment)) {
|
||||
if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
|
||||
*is_large = true; // possibly
|
||||
};
|
||||
}
|
||||
#elif defined(__sun)
|
||||
if (allow_large && _mi_os_use_large_page(size, try_alignment)) {
|
||||
struct memcntl_mha cmd = {0};
|
||||
cmd.mha_pagesize = large_os_page_size;
|
||||
cmd.mha_cmd = MHA_MAPSIZE_VA;
|
||||
if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
|
||||
*is_large = true;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
if (p == NULL) {
|
||||
_mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) {
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
mi_assert_internal(commit || !allow_large);
|
||||
mi_assert_internal(try_alignment > 0);
|
||||
|
||||
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
|
||||
return unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Commit/Reset
|
||||
//---------------------------------------------
|
||||
|
||||
static void unix_mprotect_hint(int err) {
|
||||
#if defined(__linux__) && (MI_SECURE>=2) // guard page around every mimalloc page
|
||||
if (err == ENOMEM) {
|
||||
_mi_warning_message("The next warning may be caused by a low memory map limit.\n"
|
||||
" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
|
||||
" For example: sudo sysctl -w vm.max_map_count=262144\n");
|
||||
}
|
||||
#else
|
||||
MI_UNUSED(err);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
int _mi_prim_commit(void* start, size_t size, bool commit) {
|
||||
/*
|
||||
#if 0 && defined(MAP_FIXED) && !defined(__APPLE__)
|
||||
// Linux: disabled for now as mmap fixed seems much more expensive than MADV_DONTNEED (and splits VMA's?)
|
||||
if (commit) {
|
||||
// commit: just change the protection
|
||||
err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) { err = errno; }
|
||||
}
|
||||
else {
|
||||
// decommit: use mmap with MAP_FIXED to discard the existing memory (and reduce rss)
|
||||
const int fd = mi_unix_mmap_fd();
|
||||
void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
|
||||
if (p != start) { err = errno; }
|
||||
}
|
||||
#else
|
||||
*/
|
||||
int err = 0;
|
||||
if (commit) {
|
||||
// commit: ensure we can access the area
|
||||
err = mprotect(start, size, (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) { err = errno; }
|
||||
}
|
||||
else {
|
||||
#if defined(MADV_DONTNEED) && MI_DEBUG == 0 && MI_SECURE == 0
|
||||
// decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
|
||||
// (on the other hand, MADV_FREE would be good enough.. it is just not reflected in the stats :-( )
|
||||
err = unix_madvise(start, size, MADV_DONTNEED);
|
||||
#else
|
||||
// decommit: just disable access (also used in debug and secure mode to trap on illegal access)
|
||||
err = mprotect(start, size, PROT_NONE);
|
||||
if (err != 0) { err = errno; }
|
||||
#endif
|
||||
}
|
||||
unix_mprotect_hint(err);
|
||||
return err;
|
||||
}
|
||||
|
||||
int _mi_prim_reset(void* start, size_t size) {
|
||||
#if defined(MADV_FREE)
|
||||
static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);
|
||||
int oadvice = (int)mi_atomic_load_relaxed(&advice);
|
||||
int err;
|
||||
while ((err = unix_madvise(start, size, oadvice)) != 0 && errno == EAGAIN) { errno = 0; };
|
||||
if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) {
|
||||
// if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
|
||||
mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);
|
||||
err = unix_madvise(start, size, MADV_DONTNEED);
|
||||
}
|
||||
#else
|
||||
int err = unix_madvise(start, csize, MADV_DONTNEED);
|
||||
#endif
|
||||
return err;
|
||||
}
|
||||
|
||||
int _mi_prim_protect(void* start, size_t size, bool protect) {
|
||||
int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) { err = errno; }
|
||||
unix_mprotect_hint(err);
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Huge page allocation
|
||||
//---------------------------------------------
|
||||
|
||||
#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__)
|
||||
|
||||
#include <sys/syscall.h>
|
||||
|
||||
#ifndef MPOL_PREFERRED
|
||||
#define MPOL_PREFERRED 1
|
||||
#endif
|
||||
|
||||
#if defined(SYS_mbind)
|
||||
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
|
||||
return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);
|
||||
}
|
||||
#else
|
||||
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
|
||||
MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) {
|
||||
bool is_large = true;
|
||||
void* p = unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large);
|
||||
if (p == NULL) return NULL;
|
||||
if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes
|
||||
unsigned long numa_mask = (1UL << numa_node);
|
||||
// TODO: does `mbind` work correctly for huge OS pages? should we
|
||||
// use `set_mempolicy` before calling mmap instead?
|
||||
// see: <https://lkml.org/lkml/2017/2/9/875>
|
||||
long err = mi_prim_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
|
||||
if (err != 0) {
|
||||
_mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno));
|
||||
}
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
//---------------------------------------------
|
||||
// NUMA nodes
|
||||
//---------------------------------------------
|
||||
|
||||
#if defined(__linux__)
|
||||
|
||||
#include <sys/syscall.h> // getcpu
|
||||
#include <stdio.h> // access
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
#ifdef SYS_getcpu
|
||||
unsigned long node = 0;
|
||||
unsigned long ncpu = 0;
|
||||
long err = syscall(SYS_getcpu, &ncpu, &node, NULL);
|
||||
if (err != 0) return 0;
|
||||
return node;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
char buf[128];
|
||||
unsigned node = 0;
|
||||
for(node = 0; node < 256; node++) {
|
||||
// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
|
||||
snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
|
||||
if (access(buf,R_OK) != 0) break;
|
||||
}
|
||||
return (node+1);
|
||||
}
|
||||
|
||||
#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000
|
||||
|
||||
size_t mi_prim_numa_node(void) {
|
||||
domainset_t dom;
|
||||
size_t node;
|
||||
int policy;
|
||||
if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;
|
||||
for (node = 0; node < MAXMEMDOM; node++) {
|
||||
if (DOMAINSET_ISSET(node, &dom)) return node;
|
||||
}
|
||||
return 0ul;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
size_t ndomains = 0;
|
||||
size_t len = sizeof(ndomains);
|
||||
if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul;
|
||||
return ndomains;
|
||||
}
|
||||
|
||||
#elif defined(__DragonFly__)
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
// TODO: DragonFly does not seem to provide any userland means to get this information.
|
||||
return 0ul;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
size_t ncpus = 0, nvirtcoresperphys = 0;
|
||||
size_t len = sizeof(size_t);
|
||||
if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul;
|
||||
if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;
|
||||
return nvirtcoresperphys * ncpus;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc-internal.h"
|
||||
#include "mimalloc-atomic.h"
|
||||
#include "prim.h"
|
||||
|
||||
//---------------------------------------------
|
||||
// Initialize
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
|
||||
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
|
||||
config->alloc_granularity = 16;
|
||||
config->has_overcommit = false;
|
||||
config->must_free_whole = true;
|
||||
}
|
||||
|
||||
//---------------------------------------------
|
||||
// Free
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_free(void* addr, size_t size ) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
// wasi heap cannot be shrunk
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Allocation: sbrk or memory_grow
|
||||
//---------------------------------------------
|
||||
|
||||
#if defined(MI_USE_SBRK)
|
||||
static void* mi_memory_grow( size_t size ) {
|
||||
void* p = sbrk(size);
|
||||
if (p == (void*)(-1)) return NULL;
|
||||
#if !defined(__wasi__) // on wasi this is always zero initialized already (?)
|
||||
memset(p,0,size);
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
#elif defined(__wasi__)
|
||||
static void* mi_memory_grow( size_t size ) {
|
||||
size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))
|
||||
: __builtin_wasm_memory_size(0));
|
||||
if (base == SIZE_MAX) return NULL;
|
||||
return (void*)(base * _mi_os_page_size());
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
#endif
|
||||
|
||||
static void* mi_prim_mem_grow(size_t size, size_t try_alignment) {
|
||||
void* p = NULL;
|
||||
if (try_alignment <= 1) {
|
||||
// `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_lock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
p = mi_memory_grow(size);
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_unlock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
void* base = NULL;
|
||||
size_t alloc_size = 0;
|
||||
// to allocate aligned use a lock to try to avoid thread interaction
|
||||
// between getting the current size and actual allocation
|
||||
// (also, `sbrk` is not thread safe in general)
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_lock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
{
|
||||
void* current = mi_memory_grow(0); // get current size
|
||||
if (current != NULL) {
|
||||
void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space
|
||||
alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());
|
||||
base = mi_memory_grow(alloc_size);
|
||||
}
|
||||
}
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_unlock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
if (base != NULL) {
|
||||
p = mi_align_up_ptr(base, try_alignment);
|
||||
if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {
|
||||
// another thread used wasm_memory_grow/sbrk in-between and we do not have enough
|
||||
// space after alignment. Give up (and waste the space as we cannot shrink :-( )
|
||||
// (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)
|
||||
p = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (p == NULL) {
|
||||
_mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment);
|
||||
errno = ENOMEM;
|
||||
return NULL;
|
||||
}
|
||||
mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );
|
||||
return p;
|
||||
}
|
||||
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) {
|
||||
MI_UNUSED(allow_large);
|
||||
*is_large = false;
|
||||
return mi_prim_mem_grow(size, try_alignment);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Commit/Reset/Protect
|
||||
//---------------------------------------------
|
||||
|
||||
int _mi_prim_commit(void* addr, size_t size, bool commit) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(commit);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_reset(void* addr, size_t size) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Huge pages and NUMA nodes
|
||||
//---------------------------------------------
|
||||
|
||||
void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node) {
|
||||
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -0,0 +1,385 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
#include "mimalloc.h"
|
||||
#include "mimalloc-internal.h"
|
||||
#include "mimalloc-atomic.h"
|
||||
#include "prim.h"
|
||||
#include <string.h> // strerror
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(disable:4996) // strerror
|
||||
#endif
|
||||
|
||||
//---------------------------------------------
|
||||
// Dynamically bind Windows API points for portability
|
||||
//---------------------------------------------
|
||||
|
||||
// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
|
||||
// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
|
||||
// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
|
||||
// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's.
|
||||
typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {
|
||||
MiMemExtendedParameterInvalidType = 0,
|
||||
MiMemExtendedParameterAddressRequirements,
|
||||
MiMemExtendedParameterNumaNode,
|
||||
MiMemExtendedParameterPartitionHandle,
|
||||
MiMemExtendedParameterUserPhysicalHandle,
|
||||
MiMemExtendedParameterAttributeFlags,
|
||||
MiMemExtendedParameterMax
|
||||
} MI_MEM_EXTENDED_PARAMETER_TYPE;
|
||||
|
||||
typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {
|
||||
struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;
|
||||
union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;
|
||||
} MI_MEM_EXTENDED_PARAMETER;
|
||||
|
||||
typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {
|
||||
PVOID LowestStartingAddress;
|
||||
PVOID HighestEndingAddress;
|
||||
SIZE_T Alignment;
|
||||
} MI_MEM_ADDRESS_REQUIREMENTS;
|
||||
|
||||
#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010
|
||||
|
||||
#include <winternl.h>
|
||||
typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
|
||||
typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
|
||||
static PVirtualAlloc2 pVirtualAlloc2 = NULL;
|
||||
static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
|
||||
|
||||
// Similarly, GetNumaProcesorNodeEx is only supported since Windows 7
|
||||
typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;
|
||||
|
||||
typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);
|
||||
typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);
|
||||
typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);
|
||||
typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber);
|
||||
static PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL;
|
||||
static PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL;
|
||||
static PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL;
|
||||
static PGetNumaProcessorNode pGetNumaProcessorNode = NULL;
|
||||
|
||||
//---------------------------------------------
|
||||
// Enable large page support dynamically (if possible)
|
||||
//---------------------------------------------
|
||||
|
||||
static bool win_enable_large_os_pages(size_t* large_page_size)
|
||||
{
|
||||
static bool large_initialized = false;
|
||||
if (large_initialized) return (_mi_os_large_page_size() > 0);
|
||||
large_initialized = true;
|
||||
|
||||
// Try to see if large OS pages are supported
|
||||
// To use large pages on Windows, we first need access permission
|
||||
// Set "Lock pages in memory" permission in the group policy editor
|
||||
// <https://devblogs.microsoft.com/oldnewthing/20110128-00/?p=11643>
|
||||
unsigned long err = 0;
|
||||
HANDLE token = NULL;
|
||||
BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
|
||||
if (ok) {
|
||||
TOKEN_PRIVILEGES tp;
|
||||
ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);
|
||||
if (ok) {
|
||||
tp.PrivilegeCount = 1;
|
||||
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
|
||||
ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
|
||||
if (ok) {
|
||||
err = GetLastError();
|
||||
ok = (err == ERROR_SUCCESS);
|
||||
if (ok && large_page_size != NULL) {
|
||||
*large_page_size = GetLargePageMinimum();
|
||||
}
|
||||
}
|
||||
}
|
||||
CloseHandle(token);
|
||||
}
|
||||
if (!ok) {
|
||||
if (err == 0) err = GetLastError();
|
||||
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
|
||||
}
|
||||
return (ok!=0);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Initialize
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config )
|
||||
{
|
||||
config->has_overcommit = false;
|
||||
config->must_free_whole = true;
|
||||
// get the page size
|
||||
SYSTEM_INFO si;
|
||||
GetSystemInfo(&si);
|
||||
if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }
|
||||
if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; }
|
||||
// get the VirtualAlloc2 function
|
||||
HINSTANCE hDll;
|
||||
hDll = LoadLibrary(TEXT("kernelbase.dll"));
|
||||
if (hDll != NULL) {
|
||||
// use VirtualAlloc2FromApp if possible as it is available to Windows store apps
|
||||
pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
|
||||
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
|
||||
FreeLibrary(hDll);
|
||||
}
|
||||
// NtAllocateVirtualMemoryEx is used for huge page allocation
|
||||
hDll = LoadLibrary(TEXT("ntdll.dll"));
|
||||
if (hDll != NULL) {
|
||||
pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
|
||||
FreeLibrary(hDll);
|
||||
}
|
||||
// Try to use Win7+ numa API
|
||||
hDll = LoadLibrary(TEXT("kernel32.dll"));
|
||||
if (hDll != NULL) {
|
||||
pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx");
|
||||
pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx");
|
||||
pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, "GetNumaNodeProcessorMaskEx");
|
||||
pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNode");
|
||||
FreeLibrary(hDll);
|
||||
}
|
||||
if (mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
|
||||
win_enable_large_os_pages(&config->large_page_size);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Free
|
||||
//---------------------------------------------
|
||||
|
||||
void _mi_prim_free(void* addr, size_t size ) {
|
||||
DWORD errcode = 0;
|
||||
bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
|
||||
if (err) { errcode = GetLastError(); }
|
||||
if (errcode == ERROR_INVALID_ADDRESS) {
|
||||
// In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside
|
||||
// the memory region returned by VirtualAlloc; in that case we need to free using
|
||||
// the start of the region.
|
||||
MEMORY_BASIC_INFORMATION info = { 0 };
|
||||
VirtualQuery(addr, &info, sizeof(info));
|
||||
if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)MI_SEGMENT_SIZE) {
|
||||
errcode = 0;
|
||||
err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);
|
||||
if (err) { errcode = GetLastError(); }
|
||||
}
|
||||
}
|
||||
if (errcode != 0) {
|
||||
_mi_warning_message("unable to release OS memory: error code 0x%x, addr: %p, size: %zu\n", errcode, addr, size);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// VirtualAlloc
|
||||
//---------------------------------------------
|
||||
|
||||
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
#if (MI_INTPTR_SIZE >= 8)
|
||||
// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
|
||||
if (addr == NULL) {
|
||||
void* hint = _mi_os_get_aligned_hint(try_alignment,size);
|
||||
if (hint != NULL) {
|
||||
void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);
|
||||
if (p != NULL) return p;
|
||||
_mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags);
|
||||
// fall through on error
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// on modern Windows try use VirtualAlloc2 for aligned allocation
|
||||
if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
|
||||
MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
|
||||
reqs.Alignment = try_alignment;
|
||||
MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
|
||||
param.Type.Type = MiMemExtendedParameterAddressRequirements;
|
||||
param.Arg.Pointer = &reqs;
|
||||
void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1);
|
||||
if (p != NULL) return p;
|
||||
_mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
|
||||
// fall through on error
|
||||
}
|
||||
// last resort
|
||||
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
|
||||
}
|
||||
|
||||
static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
|
||||
mi_assert_internal(!(large_only && !allow_large));
|
||||
static _Atomic(size_t) large_page_try_ok; // = 0;
|
||||
void* p = NULL;
|
||||
// Try to allocate large OS pages (2MiB) if allowed or required.
|
||||
if ((large_only || _mi_os_use_large_page(size, try_alignment))
|
||||
&& allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) {
|
||||
size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
|
||||
if (!large_only && try_ok > 0) {
|
||||
// if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
|
||||
// therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times.
|
||||
mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);
|
||||
}
|
||||
else {
|
||||
// large OS pages must always reserve and commit.
|
||||
*is_large = true;
|
||||
p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
|
||||
if (large_only) return p;
|
||||
// fall back to non-large page allocation on error (`p == NULL`).
|
||||
if (p == NULL) {
|
||||
mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fall back to regular page allocation
|
||||
if (p == NULL) {
|
||||
*is_large = ((flags&MEM_LARGE_PAGES) != 0);
|
||||
p = win_virtual_alloc_prim(addr, size, try_alignment, flags);
|
||||
}
|
||||
if (p == NULL) {
|
||||
_mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large) {
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
mi_assert_internal(commit || !allow_large);
|
||||
mi_assert_internal(try_alignment > 0);
|
||||
int flags = MEM_RESERVE;
|
||||
if (commit) { flags |= MEM_COMMIT; }
|
||||
return win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Commit/Reset/Protect
|
||||
//---------------------------------------------
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(disable:6250) // suppress warning calling VirtualFree without MEM_RELEASE (for decommit)
|
||||
#endif
|
||||
|
||||
int _mi_prim_commit(void* addr, size_t size, bool commit) {
|
||||
if (commit) {
|
||||
void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
|
||||
return (p == addr ? 0 : (int)GetLastError());
|
||||
}
|
||||
else {
|
||||
BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);
|
||||
return (ok ? 0 : (int)GetLastError());
|
||||
}
|
||||
}
|
||||
|
||||
int _mi_prim_reset(void* addr, size_t size) {
|
||||
void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
|
||||
mi_assert_internal(p == addr);
|
||||
#if 1
|
||||
if (p == addr && addr != NULL) {
|
||||
VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory from the working set
|
||||
}
|
||||
#endif
|
||||
return (p == addr ? 0 : (int)GetLastError());
|
||||
}
|
||||
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect) {
|
||||
DWORD oldprotect = 0;
|
||||
BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);
|
||||
return (ok ? 0 : (int)GetLastError());
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Huge page allocation
|
||||
//---------------------------------------------
|
||||
|
||||
void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node)
|
||||
{
|
||||
const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;
|
||||
|
||||
win_enable_large_os_pages(NULL);
|
||||
|
||||
MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
|
||||
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
|
||||
static bool mi_huge_pages_available = true;
|
||||
if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) {
|
||||
params[0].Type.Type = MiMemExtendedParameterAttributeFlags;
|
||||
params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
|
||||
ULONG param_count = 1;
|
||||
if (numa_node >= 0) {
|
||||
param_count++;
|
||||
params[1].Type.Type = MiMemExtendedParameterNumaNode;
|
||||
params[1].Arg.ULong = (unsigned)numa_node;
|
||||
}
|
||||
SIZE_T psize = size;
|
||||
void* base = addr;
|
||||
NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count);
|
||||
if (err == 0 && base != NULL) {
|
||||
return base;
|
||||
}
|
||||
else {
|
||||
// fall back to regular large pages
|
||||
mi_huge_pages_available = false; // don't try further huge pages
|
||||
_mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err);
|
||||
}
|
||||
}
|
||||
// on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation
|
||||
if (pVirtualAlloc2 != NULL && numa_node >= 0) {
|
||||
params[0].Type.Type = MiMemExtendedParameterNumaNode;
|
||||
params[0].Arg.ULong = (unsigned)numa_node;
|
||||
return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
|
||||
}
|
||||
|
||||
// otherwise use regular virtual alloc on older windows
|
||||
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------
|
||||
// Numa nodes
|
||||
//---------------------------------------------
|
||||
|
||||
size_t _mi_prim_numa_node(void) {
|
||||
USHORT numa_node = 0;
|
||||
if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {
|
||||
// Extended API is supported
|
||||
MI_PROCESSOR_NUMBER pnum;
|
||||
(*pGetCurrentProcessorNumberEx)(&pnum);
|
||||
USHORT nnode = 0;
|
||||
BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode);
|
||||
if (ok) { numa_node = nnode; }
|
||||
}
|
||||
else if (pGetNumaProcessorNode != NULL) {
|
||||
// Vista or earlier, use older API that is limited to 64 processors. Issue #277
|
||||
DWORD pnum = GetCurrentProcessorNumber();
|
||||
UCHAR nnode = 0;
|
||||
BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode);
|
||||
if (ok) { numa_node = nnode; }
|
||||
}
|
||||
return numa_node;
|
||||
}
|
||||
|
||||
size_t _mi_prim_numa_node_count(void) {
|
||||
ULONG numa_max = 0;
|
||||
GetNumaHighestNodeNumber(&numa_max);
|
||||
// find the highest node number that has actual processors assigned to it. Issue #282
|
||||
while(numa_max > 0) {
|
||||
if (pGetNumaNodeProcessorMaskEx != NULL) {
|
||||
// Extended API is supported
|
||||
GROUP_AFFINITY affinity;
|
||||
if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) {
|
||||
if (affinity.Mask != 0) break; // found the maximum non-empty node
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Vista or earlier, use older API that is limited to 64 processors.
|
||||
ULONGLONG mask;
|
||||
if (GetNumaNodeProcessorMask((UCHAR)numa_max, &mask)) {
|
||||
if (mask != 0) break; // found the maximum non-empty node
|
||||
};
|
||||
}
|
||||
// max node was invalid or had no processor assigned, try again
|
||||
numa_max--;
|
||||
}
|
||||
return ((size_t)numa_max + 1);
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
// Select the implementation of the primitives
|
||||
// depending on the OS.
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include "prim-windows.c" // VirtualAlloc (Windows)
|
||||
#elif defined(__wasi__)
|
||||
#define MI_USE_SBRK
|
||||
#include "prim-wasi.h" // memory-grow or sbrk (Wasm)
|
||||
#else
|
||||
#include "prim-unix.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)
|
||||
#endif
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef MIMALLOC_PRIM_H
|
||||
#define MIMALLOC_PRIM_H
|
||||
|
||||
// note: on all primitive functions, we always get:
|
||||
// addr != NULL and page aligned
|
||||
// size > 0 and page aligned
|
||||
//
|
||||
|
||||
// OS memory configuration
|
||||
typedef struct mi_os_mem_config_s {
|
||||
size_t page_size; // 4KiB
|
||||
size_t large_page_size; // 2MiB
|
||||
size_t alloc_granularity; // smallest allocation size (on Windows 64KiB)
|
||||
bool has_overcommit; // can we reserve more memory than can be actually committed?
|
||||
bool must_free_whole; // must allocated blocks free as a whole (false for mmap, true for VirtualAlloc)
|
||||
} mi_os_mem_config_t;
|
||||
|
||||
// Initialize
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config );
|
||||
|
||||
// Free OS memory
|
||||
// pre: addr != NULL, size > 0
|
||||
void _mi_prim_free(void* addr, size_t size );
|
||||
|
||||
// Allocate OS memory.
|
||||
// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.
|
||||
// return NULL on error.
|
||||
// pre: !commit => !allow_large
|
||||
// try_alignment >= _mi_os_page_size() and a power of 2
|
||||
void* _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large);
|
||||
|
||||
// Commit memory. Returns error code or 0 on success.
|
||||
int _mi_prim_commit(void* addr, size_t size, bool commit);
|
||||
|
||||
// Reset memory. The range keeps being accessible but the content might be reset.
|
||||
// Returns error code or 0 on success.
|
||||
int _mi_prim_reset(void* addr, size_t size);
|
||||
|
||||
// Protect memory. Returns error code or 0 on success.
|
||||
int _mi_prim_protect(void* addr, size_t size, bool protect);
|
||||
|
||||
// Allocate huge (1GiB) pages possibly associated with a NUMA node.
|
||||
// pre: size > 0 and a multiple of 1GiB.
|
||||
// addr is either NULL or an address hint.
|
||||
// numa_node is either negative (don't care), or a numa node number.
|
||||
void* _mi_prim_alloc_huge_os_pages(void* addr, size_t size, int numa_node);
|
||||
|
||||
// Return the current NUMA node
|
||||
size_t _mi_prim_numa_node(void);
|
||||
|
||||
// Return the number of logical NUMA nodes
|
||||
size_t _mi_prim_numa_node_count(void);
|
||||
|
||||
|
||||
#endif // MIMALLOC_PRIM_H
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
This is the portability layer where all primitives needed from the OS are defined.
|
||||
|
||||
- `prim.h`: API definition
|
||||
- `prim.c`: Selects one of `prim-unix.c`, `prim-wasi.c`, or `prim-windows.c` depending on the host platform.
|
||||
|
||||
Note: still work in progress, there may be other places in the sources that still depend on OS ifdef's.
|
||||
Loading…
Reference in New Issue