mirrors/mimalloc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge from dev-exp

Browse Source
This commit is contained in:
daan 2019-11-03 13:37:03 -08:00
commit 2b005addd3
8 changed files with 117 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -1,7 +1,5 @@
cmake_minimum_required(VERSION 3.0)
project(libmimalloc C CXX)
include("cmake/mimalloc-config-version.cmake")
include("CheckIncludeFile")
set(CMAKE_C_STANDARD 11)
set(CMAKE_CXX_STANDARD 17)
@ -15,6 +13,9 @@ option(MI_SECURE "Use security mitigations (like guard pages and rand
option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF)
option(MI_BUILD_TESTS "Build test executables" ON)
include("cmake/mimalloc-config-version.cmake")
include("CheckIncludeFile")
set(mi_install_dir "lib/mimalloc-${mi_version}")
set(mi_sources

1
include/mimalloc.h
View File

@ -277,6 +277,7 @@ typedef enum mi_option_e {
mi_option_segment_reset,
mi_option_os_tag,
mi_option_max_numa_node,
mi_option_max_errors,
_mi_option_last
} mi_option_t;

36
src/arena.c
View File

@ -27,7 +27,10 @@ with on-demand coalescing.
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld);
//int _mi_os_alloc_huge_os_pages(size_t pages, double max_secs, void** pstart, size_t* pages_reserved, size_t* psize) mi_attr_noexcept;
void _mi_os_free(void* p, size_t size, mi_stats_t* stats);
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, size_t* psize);
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, double max_secs, size_t* pages_reserved, size_t* psize);
void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats);
int _mi_os_numa_node_count(void);
@ -235,12 +238,12 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n
void* p = mi_arena_alloc(arena, needed_bcount, is_zero, &block_index);
if (p != NULL) {
mi_assert_internal(block_index != SIZE_MAX);
#if MI_DEBUG>=1
#if MI_DEBUG>=1
_Atomic(mi_block_info_t)* block = &arena->blocks[block_index];
mi_block_info_t binfo = mi_atomic_read(block);
mi_assert_internal(mi_block_is_in_use(binfo));
mi_assert_internal(mi_block_count(binfo) >= needed_bcount);
#endif
#endif
*memid = mi_memid_create(arena_index, block_index);
*commit = true; // TODO: support commit on demand?
*large = arena->is_large;
@ -383,18 +386,23 @@ static bool mi_arena_add(mi_arena_t* arena) {
// reserve at a specific numa node
int mi_reserve_huge_os_pages_at(size_t pages, int numa_node) mi_attr_noexcept {
size_t hsize = 0;
if (pages==0) return 0;
if (numa_node < -1) numa_node = -1;
if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();
void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, &hsize);
if (p==NULL) return ENOMEM;
_mi_verbose_message("reserved %zu huge (1GiB) pages\n", pages);
size_t hsize = 0;
size_t pages_reserved = 0;
void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, (double)pages / 2.0, &pages_reserved, &hsize);
if (p==NULL || pages_reserved==0) {
_mi_warning_message("failed to reserve %zu gb huge pages\n", pages);
return ENOMEM;
}
_mi_verbose_message("reserved %zu gb huge pages\n", pages_reserved);
size_t bcount = hsize / MI_ARENA_BLOCK_SIZE;
size_t asize = sizeof(mi_arena_t) + (bcount*sizeof(mi_block_info_t)); // one too much
mi_arena_t* arena = (mi_arena_t*)_mi_os_alloc(asize, &_mi_stats_main); // TODO: can we avoid allocating from the OS?
if (arena == NULL) {
_mi_os_free(p, hsize, &_mi_stats_main);
_mi_os_free_huge_pages(p, hsize, &_mi_stats_main);
return ENOMEM;
}
arena->block_count = bcount;
@ -416,18 +424,20 @@ int mi_reserve_huge_os_pages_interleave(size_t pages) mi_attr_noexcept {
// pages per numa node
int numa_count = _mi_os_numa_node_count();
if (numa_count <= 0) numa_count = 1;
size_t pages_per = pages / numa_count;
if (pages_per == 0) pages_per = 1;
const size_t pages_per = pages / numa_count;
const size_t pages_mod = pages % numa_count;
// reserve evenly among numa nodes
for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) {
int err = mi_reserve_huge_os_pages_at((pages_per > pages ? pages : pages_per), numa_node);
size_t node_pages = pages_per; // can be 0
if ((size_t)numa_node < pages_mod) node_pages++;
int err = mi_reserve_huge_os_pages_at(node_pages, numa_node);
if (err) return err;
if (pages < pages_per) {
if (pages < node_pages) {
pages = 0;
}
else {
pages -= pages_per;
pages -= node_pages;
}
}

2
src/heap.c
View File

@ -45,7 +45,7 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void
}
#if MI_DEBUG>1
#if MI_DEBUG>=3
static bool _mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
UNUSED(arg1);
UNUSED(arg2);

11
src/options.c
View File

@ -14,6 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include <ctype.h> // toupper
#include <stdarg.h>
static uintptr_t mi_max_error_count = 16; // stop outputting errors after this
static void mi_add_stderr_output();
int mi_version(void) mi_attr_noexcept {
@ -65,7 +67,8 @@ static mi_option_desc_t options[_mi_option_last] =
{ 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit pages when not eager committed
{ 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit)
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
{ 256, UNINIT, MI_OPTION(max_numa_node) } // maximum allowed numa node
{ 256, UNINIT, MI_OPTION(max_numa_node) }, // maximum allowed numa node
{ 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output
};
static void mi_option_init(mi_option_desc_t* desc);
@ -82,6 +85,7 @@ void _mi_options_init(void) {
_mi_verbose_message("option '%s': %ld\n", desc->name, desc->value);
}
}
mi_max_error_count = mi_option_get(mi_option_max_errors);
}
long mi_option_get(mi_option_t option) {
@ -217,7 +221,6 @@ static void mi_add_stderr_output() {
// --------------------------------------------------------
// Messages, all end up calling `_mi_fputs`.
// --------------------------------------------------------
#define MAX_ERROR_COUNT (10)
static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings
// When overriding malloc, we may recurse into mi_vfprintf if an allocation
@ -272,7 +275,7 @@ void _mi_verbose_message(const char* fmt, ...) {
void _mi_error_message(const char* fmt, ...) {
if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return;
if (mi_atomic_increment(&error_count) > mi_max_error_count) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(NULL, "mimalloc: error: ", fmt, args);
@ -282,7 +285,7 @@ void _mi_error_message(const char* fmt, ...) {
void _mi_warning_message(const char* fmt, ...) {
if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return;
if (mi_atomic_increment(&error_count) > mi_max_error_count) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(NULL, "mimalloc: warning: ", fmt, args);

119
src/os.c
View File

@ -338,7 +338,8 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
lflags |= MAP_HUGETLB;
#endif
#ifdef MAP_HUGE_1GB
if ((size % GiB) == 0) {
static bool mi_huge_pages_available = true;
if ((size % GiB) == 0 && mi_huge_pages_available) {
lflags |= MAP_HUGE_1GB;
}
else
@ -357,6 +358,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) {
mi_huge_pages_available = false; // don't try huge 1GiB pages again
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno);
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
@ -798,11 +800,11 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
mi_win_enable_large_os_pages();
void* p = NULL;
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
MEM_EXTENDED_PARAMETER params[3] = { {0,0},{0,0},{0,0} };
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
if (pNtAllocateVirtualMemoryEx != NULL) {
static bool mi_huge_pages_available = true;
if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) {
#ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
#define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE (0x10)
#endif
@ -821,7 +823,8 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
return base;
}
else {
// fall back to regular huge pages
// fall back to regular large pages
mi_huge_pages_available = false; // don't try further huge pages
_mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err);
}
}
@ -829,20 +832,11 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
if (pVirtualAlloc2 != NULL && numa_node >= 0) {
params[0].Type = MemExtendedParameterNumaNode;
params[0].ULong = (unsigned)numa_node;
p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
}
else
#endif
// use regular virtual alloc on older windows
{
p = VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
if (p == NULL) {
DWORD winerr = GetLastError();
_mi_warning_message("failed to allocate huge OS pages (size %zu) (windows error %d%s)\n", size, winerr, (winerr==1450 ? " (insufficient resources)" : ""));
}
return p;
// otherwise use regular virtual alloc on older windows
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
#elif defined(MI_OS_USE_MMAP) && (MI_INTPTR_SIZE >= 8)
@ -879,44 +873,91 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
// To ensure proper alignment, use our own area for huge OS pages
static _Atomic(uintptr_t) mi_huge_start; // = 0
// Allocate MI_SEGMENT_SIZE aligned huge pages
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, size_t* psize) {
if (psize != NULL) *psize = 0;
// Claim an aligned address range for huge pages
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
if (total_size != NULL) *total_size = 0;
const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;
// Find a new aligned address for the huge pages
uintptr_t start = 0;
uintptr_t end = 0;
uintptr_t expected;
do {
start = expected = mi_atomic_read_relaxed(&mi_huge_start);
start = expected = mi_atomic_read_relaxed(&mi_huge_start);
if (start == 0) {
// Initialize the start address after the 32TiB area
start = ((uintptr_t)32 << 40); // 32TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&_mi_os_alloc_huge_os_pages);
start = ((uintptr_t)32 << 40); // 32TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_os_claim_huge_pages);
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB
#endif
#endif
}
end = start + size;
mi_assert_internal(end % MI_SEGMENT_SIZE == 0);
} while (!mi_atomic_cas_strong(&mi_huge_start, end, expected));
// And allocate
void* p = mi_os_alloc_huge_os_pagesx((void*)start, size, numa_node);
if (p == NULL) {
return NULL;
}
_mi_stat_increase(&_mi_stats_main.committed, size);
_mi_stat_increase(&_mi_stats_main.reserved, size);
if ((uintptr_t)p % MI_SEGMENT_SIZE != 0) { // must be aligned
_mi_warning_message("huge page area was not aligned\n");
_mi_os_free(p,size,&_mi_stats_main);
return NULL;
}
if (total_size != NULL) *total_size = size;
return (uint8_t*)start;
}
// Allocate MI_SEGMENT_SIZE aligned huge pages
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, double max_secs, size_t* pages_reserved, size_t* psize) {
if (psize != NULL) *psize = 0;
if (pages_reserved != NULL) *pages_reserved = 0;
size_t size = 0;
uint8_t* start = mi_os_claim_huge_pages(pages, &size);
if (psize != NULL) *psize = size;
return p;
// Allocate one page at the time but try to place them contiguously
// We allocate one page at the time to be able to abort if it takes too long
// or to at least allocate as many as available on the system.
double start_t = _mi_clock_start();
size_t page;
for (page = 0; page < pages; page++) {
// allocate a page
void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
void* p = mi_os_alloc_huge_os_pagesx(addr, MI_HUGE_OS_PAGE_SIZE, numa_node);
// Did we succeed at a contiguous address?
if (p != addr) {
// no success, issue a warning and break
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge page %zu at 0x%p\n", page, addr);
_mi_os_free(p, MI_HUGE_OS_PAGE_SIZE, &_mi_stats_main);
}
break;
}
// success, record it
_mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE);
_mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE);
// check for timeout
double elapsed = _mi_clock_end(start_t);
if (page >= 1) {
double estimate = ((elapsed / (double)(page+1)) * (double)pages);
if (estimate > 1.5*max_secs) { // seems like we are going to timeout, break
elapsed = max_secs + 1.0;
}
}
if (elapsed > max_secs) {
_mi_warning_message("huge page allocation timed out\n");
break;
}
}
mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size);
if (pages_reserved != NULL) *pages_reserved = page;
if (psize != NULL) *psize = page * MI_HUGE_OS_PAGE_SIZE;
return (page == 0 ? NULL : start);
}
// free every huge page in a range individually (as we allocated per page)
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
_mi_os_free(base, MI_HUGE_OS_PAGE_SIZE, stats);
size -= MI_HUGE_OS_PAGE_SIZE;
}
}
/* ----------------------------------------------------------------------------

2
src/page.c
View File

@ -38,7 +38,7 @@ static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_sta
static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_stats_t* stats);
#if (MI_DEBUG>1)
#if (MI_DEBUG>=3)
static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {
size_t count = 0;
while (head != NULL) {

3
src/segment.c
View File

@ -39,7 +39,6 @@ static const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment) {
return &segment->slices[segment->slice_entries];
}
static uint8_t* mi_slice_start(const mi_slice_t* slice) {
mi_segment_t* segment = _mi_ptr_segment(slice);
mi_assert_internal(slice >= segment->slices && slice < mi_segment_slices_end(segment));
@ -136,7 +135,7 @@ static void mi_span_queue_delete(mi_span_queue_t* sq, mi_slice_t* slice) {
Invariant checking
----------------------------------------------------------- */
#if (MI_DEBUG > 1)
#if (MI_DEBUG>=3)
static bool mi_span_queue_contains(mi_span_queue_t* sq, mi_slice_t* slice) {
for (mi_slice_t* s = sq->first; s != NULL; s = s->next) {
if (s==slice) return true;