mirrors/mimalloc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove reset delay slots; add reset tracking per page and segment

Browse Source
This commit is contained in:
Daan Leijen 2019-11-20 14:55:12 -08:00
parent 30e2c54adb
commit 211f1aa519
11 changed files with 443 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/mimalloc-internal.h
View File

@ -59,7 +59,7 @@ size_t _mi_os_good_alloc_size(size_t size);
// memory.c
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* id, mi_os_tld_t* tld);
void _mi_mem_free(void* p, size_t size, size_t id, mi_os_tld_t* tld);
void _mi_mem_free(void* p, size_t size, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld);
bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld);
bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld);
@ -75,7 +75,7 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t*
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld);
void _mi_segment_thread_collect(mi_segments_tld_t* tld);
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size); // page start for any page
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size, size_t* pre_size); // page start for any page
// "page.c"
void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc;
@ -297,7 +297,9 @@ static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const
// Quick page start for initialized pages
static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) {
return _mi_segment_page_start(segment, page, page->block_size, page_size);
const size_t bsize = page->block_size;
mi_assert_internal(bsize > 0 && (bsize%sizeof(void*)) == 0);
return _mi_segment_page_start(segment, page, bsize, page_size, NULL);
}
// Get the page containing the pointer

28
include/mimalloc-types.h
View File

@ -384,31 +384,12 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount)
// ------------------------------------------------------
// Delay slots (to avoid expensive OS calls)
// ------------------------------------------------------
typedef int64_t mi_msecs_t;
#define MI_RESET_DELAY_SLOTS (256)
typedef struct mi_delay_slot_s {
mi_msecs_t expire;
uint8_t* addr;
size_t size;
} mi_delay_slot_t;
typedef struct mi_delay_slots_s {
size_t capacity; // always `MI_RESET_DELAY_SLOTS`
size_t count; // current slots used (`<= capacity`)
mi_delay_slot_t slots[MI_RESET_DELAY_SLOTS];
} mi_delay_slots_t;
// ------------------------------------------------------
// Thread Local data
// ------------------------------------------------------
typedef int64_t mi_msecs_t;
// Queue of segments
typedef struct mi_segment_queue_s {
mi_segment_t* first;
@ -417,9 +398,8 @@ typedef struct mi_segment_queue_s {
// OS thread local data
typedef struct mi_os_tld_s {
size_t region_idx; // start point for next allocation
mi_delay_slots_t* reset_delay; // delay slots for OS reset operations
mi_stats_t* stats; // points to tld stats
size_t region_idx; // start point for next allocation
mi_stats_t* stats; // points to tld stats
} mi_os_tld_t;
// Segments thread local data

3
include/mimalloc.h
View File

@ -272,8 +272,9 @@ typedef enum mi_option_e {
mi_option_segment_cache,
mi_option_page_reset,
mi_option_segment_reset,
mi_option_eager_commit_delay,
mi_option_reset_decommits,
mi_option_eager_commit_delay,
mi_option_reset_delay,
mi_option_use_numa_nodes,
mi_option_os_tag,
mi_option_max_errors,

8
src/arena.c
View File

@ -107,7 +107,7 @@ static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t*
size_t idx = mi_atomic_read(&arena->search_idx); // start from last search
for (size_t visited = 0; visited < fcount; visited++, idx++) {
if (idx >= fcount) idx = 0; // wrap around
if (mi_bitmap_try_claim_field(arena->blocks_inuse, idx, blocks, bitmap_idx)) {
if (mi_bitmap_try_find_claim_field(arena->blocks_inuse, idx, blocks, bitmap_idx)) {
mi_atomic_write(&arena->search_idx, idx); // start search from here next time
return true;
}
@ -137,9 +137,9 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n
}
else if (commit) {
// ensure commit now
bool any_zero;
mi_bitmap_claim(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_zero);
if (any_zero) {
bool any_uncommitted;
mi_bitmap_claim(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted);
if (any_uncommitted) {
bool commit_zero;
_mi_os_commit(p, needed_bcount * MI_ARENA_BLOCK_SIZE, &commit_zero, tld->stats);
if (commit_zero) *is_zero = true;

54
src/bitmap.inc.c
View File

@ -104,9 +104,29 @@ static inline size_t mi_bsr(uintptr_t x) {
Claim a bit sequence atomically
----------------------------------------------------------- */
// Try to atomically claim a sequence of `count` bits at in `idx`
// in the bitmap field. Returns `true` on success.
static inline bool mi_bitmap_try_claim_field(mi_bitmap_t bitmap, size_t bitmap_fields, const size_t count, mi_bitmap_index_t bitmap_idx) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
mi_assert_internal(bitidx + count <= MI_BITMAP_FIELD_BITS);
mi_bitmap_field_t field = mi_atomic_read_relaxed(&bitmap[idx]);
if ((field & mask) == 0) { // free?
if (mi_atomic_cas_strong(&bitmap[idx], (field|mask), field)) {
// claimed!
return true;
}
}
return false;
}
// Try to atomically claim a sequence of `count` bits in a single
// field at `idx` in `bitmap`. Returns `true` on success.
static inline bool mi_bitmap_try_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx)
static inline bool mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx)
{
mi_assert_internal(bitmap_idx != NULL);
volatile _Atomic(uintptr_t)* field = &bitmap[idx];
@ -160,9 +180,9 @@ static inline bool mi_bitmap_try_claim_field(mi_bitmap_t bitmap, size_t idx, con
// Find `count` bits of 0 and set them to 1 atomically; returns `true` on success.
// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never span fields.
static inline bool mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t* bitmap_idx) {
static inline bool mi_bitmap_try_find_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t* bitmap_idx) {
for (size_t idx = 0; idx < bitmap_fields; idx++) {
if (mi_bitmap_try_claim_field(bitmap, idx, count, bitmap_idx)) {
if (mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) {
return true;
}
}
@ -170,39 +190,51 @@ static inline bool mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields,
}
// Set `count` bits at `bitmap_idx` to 0 atomically
// Returns `true` if all `count` bits were 1 previously
// Returns `true` if all `count` bits were 1 previously.
static inline bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
mi_assert_internal((bitmap[idx] & mask) == mask);
// mi_assert_internal((bitmap[idx] & mask) == mask);
uintptr_t prev = mi_atomic_and(&bitmap[idx], ~mask);
return ((prev & mask) == mask);
}
// Set `count` bits at `bitmap_idx` to 1 atomically
// Returns `true` if all `count` bits were 0 previously
// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.
static inline bool mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
// mi_assert_internal((bitmap[idx] & mask) == 0);
//mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0);
uintptr_t prev = mi_atomic_or(&bitmap[idx], mask);
if (any_zero != NULL) *any_zero = ((prev & mask) != mask);
return ((prev & mask) == 0);
}
// Returns `true` if all `count` bits were 1
static inline bool mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
// Returns `true` if all `count` bits were 1. `any_ones` is `true` if there was at least one bit set to one.
static inline bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
// mi_assert_internal((bitmap[idx] & mask) == 0);
return ((mi_atomic_read(&bitmap[idx]) & mask) == mask);
mi_bitmap_field_t field = mi_atomic_read_relaxed(&bitmap[idx]);
if (any_ones != NULL) *any_ones = ((field & mask) != 0);
return ((field & mask) == mask);
}
static inline bool mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
return mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, NULL);
}
static inline bool mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
bool any_ones;
mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, &any_ones);
return any_ones;
}
#endif

11
src/init.c
View File

@ -97,13 +97,11 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
#define tld_main_stats ((mi_stats_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,stats)))
#define tld_main_os ((mi_os_tld_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,os)))
static mi_delay_slots_t tld_reset_delay_main = { MI_RESET_DELAY_SLOTS, 0, { {0,NULL,0} } };
static mi_tld_t tld_main = {
0, false,
&_mi_heap_main,
{ { NULL, NULL }, {NULL ,NULL}, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats, tld_main_os }, // segments
{ 0, &tld_reset_delay_main, tld_main_stats }, // os
{ 0, tld_main_stats }, // os
{ MI_STATS_NULL } // stats
};
@ -194,8 +192,7 @@ uintptr_t _mi_random_init(uintptr_t seed /* can be zero */) {
typedef struct mi_thread_data_s {
mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
mi_tld_t tld;
mi_delay_slots_t reset_delay;
mi_tld_t tld;
} mi_thread_data_t;
// Initialize the thread local default heap, called from `mi_thread_init`
@ -215,7 +212,6 @@ static bool _mi_heap_init(void) {
}
mi_tld_t* tld = &td->tld;
mi_heap_t* heap = &td->heap;
mi_delay_slots_t* reset_delay = &td->reset_delay;
memcpy(heap, &_mi_heap_empty, sizeof(*heap));
heap->thread_id = _mi_thread_id();
heap->random = _mi_random_init(heap->thread_id);
@ -226,9 +222,6 @@ static bool _mi_heap_init(void) {
tld->segments.stats = &tld->stats;
tld->segments.os = &tld->os;
tld->os.stats = &tld->stats;
tld->os.reset_delay = reset_delay;
memset(reset_delay, 0, sizeof(*reset_delay));
reset_delay->capacity = MI_RESET_DELAY_SLOTS;
_mi_heap_set_default_direct(heap);
}
return false;

199
src/memory.c
View File

@ -54,6 +54,7 @@ void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, s
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
// Constants
#if (MI_INTPTR_SIZE==8)
#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 48KiB for the region map
@ -73,28 +74,26 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo
// Region info is a pointer to the memory region and two bits for
// its flags: is_large, and is_committed.
typedef uintptr_t mi_region_info_t;
static inline mi_region_info_t mi_region_info_create(void* start, bool is_large, bool is_committed) {
return ((uintptr_t)start | ((uintptr_t)(is_large?1:0) << 1) | (is_committed?1:0));
}
static inline void* mi_region_info_read(mi_region_info_t info, bool* is_large, bool* is_committed) {
if (is_large) *is_large = ((info&0x02) != 0);
if (is_committed) *is_committed = ((info&0x01) != 0);
return (void*)(info & ~0x03);
}
typedef union mi_region_info_u {
uintptr_t value;
struct {
bool valid;
bool is_large;
int numa_node;
};
} mi_region_info_t;
// A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with
// a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block.
typedef struct mem_region_s {
volatile _Atomic(mi_region_info_t) info; // start of the memory area (and flags)
volatile _Atomic(uintptr_t) numa_node; // associated numa node + 1 (so 0 is no association)
volatile _Atomic(uintptr_t) info; // is_large, and associated numa node + 1 (so 0 is no association)
volatile _Atomic(void*) start; // start of the memory area (and flags)
mi_bitmap_field_t in_use; // bit per in-use block
mi_bitmap_field_t dirty; // track if non-zero per block
mi_bitmap_field_t commit; // track if committed per block (if `!info.is_committed))
size_t arena_memid; // if allocated from a (huge page) arena
mi_bitmap_field_t reset; // track reset per block
volatile _Atomic(uintptr_t) arena_memid; // if allocated from a (huge page) arena-
} mem_region_t;
// The region map
@ -113,24 +112,32 @@ static size_t mi_region_block_count(size_t size) {
return _mi_divide_up(size, MI_SEGMENT_SIZE);
}
/*
// Return a rounded commit/reset size such that we don't fragment large OS pages into small ones.
static size_t mi_good_commit_size(size_t size) {
if (size > (SIZE_MAX - _mi_os_large_page_size())) return size;
return _mi_align_up(size, _mi_os_large_page_size());
}
*/
// Return if a pointer points into a region reserved by us.
bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
if (p==NULL) return false;
size_t count = mi_atomic_read_relaxed(&regions_count);
for (size_t i = 0; i < count; i++) {
uint8_t* start = (uint8_t*)mi_region_info_read( mi_atomic_read_relaxed(&regions[i].info), NULL, NULL);
uint8_t* start = (uint8_t*)mi_atomic_read_ptr_relaxed(&regions[i].start);
if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true;
}
return false;
}
static void* mi_region_blocks_start(const mem_region_t* region, mi_bitmap_index_t bit_idx) {
void* start = mi_atomic_read_ptr(&region->start);
mi_assert_internal(start != NULL);
return ((uint8_t*)start + (bit_idx * MI_SEGMENT_SIZE));
}
static size_t mi_memid_create(mem_region_t* region, mi_bitmap_index_t bit_idx) {
mi_assert_internal(bit_idx < MI_BITMAP_FIELD_BITS);
size_t idx = region - regions;
@ -142,13 +149,10 @@ static size_t mi_memid_create_from_arena(size_t arena_memid) {
return (arena_memid << 1) | 1;
}
static bool mi_memid_is_arena(size_t id) {
return ((id&1)==1);
}
static bool mi_memid_indices(size_t id, mem_region_t** region, mi_bitmap_index_t* bit_idx, size_t* arena_memid) {
if (mi_memid_is_arena(id)) {
*arena_memid = (id>>1);
static bool mi_memid_is_arena(size_t id, mem_region_t** region, mi_bitmap_index_t* bit_idx, size_t* arena_memid) {
if ((id&1)==1) {
if (arena_memid != NULL) *arena_memid = (id>>1);
return true;
}
else {
@ -159,6 +163,7 @@ static bool mi_memid_indices(size_t id, mem_region_t** region, mi_bitmap_index_t
}
}
/* ----------------------------------------------------------------------------
Allocate a region is allocated from the OS (or an arena)
-----------------------------------------------------------------------------*/
@ -187,16 +192,21 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large,
// allocated, initialize and claim the initial blocks
mem_region_t* r = &regions[idx];
r->numa_node = _mi_os_numa_node(tld) + 1;
r->arena_memid = arena_memid;
r->arena_memid = arena_memid;
mi_atomic_write(&r->in_use, 0);
mi_atomic_write(&r->dirty, (is_zero ? 0 : ~0UL));
mi_atomic_write(&r->commit, (region_commit ? ~0UL : 0));
mi_atomic_write(&r->reset, 0);
*bit_idx = 0;
mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL);
mi_atomic_write_ptr(&r->start, start);
// and share it
mi_atomic_write(&r->info, mi_region_info_create(start, region_large, region_commit)); // now make it available to others
mi_region_info_t info;
info.valid = true;
info.is_large = region_large;
info.numa_node = _mi_os_numa_node(tld);
mi_atomic_write(&r->info, info.value); // now make it available to others
*region = r;
return true;
}
@ -207,36 +217,33 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large,
static bool mi_region_is_suitable(const mem_region_t* region, int numa_node, bool allow_large ) {
// initialized at all?
mi_region_info_t info = mi_atomic_read_relaxed(&region->info);
if (info==0) return false;
mi_region_info_t info;
info.value = mi_atomic_read_relaxed(&region->info);
if (info.value==0) return false;
// numa correct
if (numa_node >= 0) { // use negative numa node to always succeed
int rnode = ((int)mi_atomic_read_relaxed(&region->numa_node)) - 1;
int rnode = info.numa_node;
if (rnode >= 0 && rnode != numa_node) return false;
}
// check allow-large
bool is_large;
bool is_committed;
mi_region_info_read(info, &is_large, &is_committed);
if (!allow_large && is_large) return false;
if (!allow_large && info.is_large) return false;
return true;
}
static bool mi_region_try_claim(size_t blocks, bool allow_large, mem_region_t** region, mi_bitmap_index_t* bit_idx, mi_os_tld_t* tld)
static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large, mem_region_t** region, mi_bitmap_index_t* bit_idx, mi_os_tld_t* tld)
{
// try all regions for a free slot
const int numa_node = (_mi_os_numa_node_count() <= 1 ? -1 : _mi_os_numa_node(tld));
// try all regions for a free slot
const size_t count = mi_atomic_read(&regions_count);
size_t idx = tld->region_idx; // Or start at 0 to reuse low addresses?
for (size_t visited = 0; visited < count; visited++, idx++) {
if (idx >= count) idx = 0; // wrap around
mem_region_t* r = &regions[idx];
if (mi_region_is_suitable(r, numa_node, allow_large)) {
if (mi_bitmap_try_claim_field(&r->in_use, 0, blocks, bit_idx)) {
if (mi_bitmap_try_find_claim_field(&r->in_use, 0, blocks, bit_idx)) {
tld->region_idx = idx; // remember the last found position
*region = r;
return true;
@ -252,8 +259,9 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo
mi_assert_internal(blocks <= MI_BITMAP_FIELD_BITS);
mem_region_t* region;
mi_bitmap_index_t bit_idx;
// first try to claim in existing regions
if (!mi_region_try_claim(blocks, *is_large, &region, &bit_idx, tld)) {
const int numa_node = (_mi_os_numa_node_count() <= 1 ? -1 : _mi_os_numa_node(tld));
// try to claim in existing regions
if (!mi_region_try_claim(numa_node, blocks, *is_large, &region, &bit_idx, tld)) {
// otherwise try to allocate a fresh region
if (!mi_region_try_alloc_os(blocks, *commit, *is_large, &region, &bit_idx, tld)) {
// out of regions or memory
@ -261,30 +269,28 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo
}
}
// found a region and claimed `blocks` at `bit_idx`
mi_assert_internal(region != NULL);
mi_assert_internal(mi_bitmap_is_claimed(&region->in_use, 1, blocks, bit_idx));
mi_region_info_t info = mi_atomic_read(&region->info);
bool region_is_committed = false;
bool region_is_large = false;
void* start = mi_region_info_read(info, &region_is_large, &region_is_committed);
mi_assert_internal(!(region_is_large && !*is_large));
mi_region_info_t info;
info.value = mi_atomic_read(&region->info);
void* start = mi_atomic_read_ptr(&region->start);
mi_assert_internal(!(info.is_large && !*is_large));
mi_assert_internal(start != NULL);
*is_zero = mi_bitmap_claim(&region->dirty, 1, blocks, bit_idx, NULL);
*is_large = region_is_large;
*is_zero = mi_bitmap_unclaim(&region->dirty, 1, blocks, bit_idx);
*is_large = info.is_large;
*memid = mi_memid_create(region, bit_idx);
void* p = (uint8_t*)start + (mi_bitmap_index_bit_in_field(bit_idx) * MI_SEGMENT_SIZE);
if (region_is_committed) {
// always committed
*commit = true;
}
else if (*commit) {
// commit
if (*commit) {
// ensure commit
bool any_zero;
mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, &any_zero);
if (any_zero) {
bool any_uncommitted;
mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, &any_uncommitted);
if (any_uncommitted) {
bool commit_zero;
_mi_mem_commit(p, blocks * MI_SEGMENT_SIZE, &commit_zero, tld);
if (commit_zero) *is_zero = true;
@ -294,6 +300,21 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo
// no need to commit, but check if already fully committed
*commit = mi_bitmap_is_claimed(&region->commit, 1, blocks, bit_idx);
}
mi_assert_internal(mi_bitmap_is_claimed(&region->commit, 1, blocks, bit_idx));
// unreset reset blocks
if (mi_bitmap_is_any_claimed(&region->reset, 1, blocks, bit_idx)) {
mi_assert_internal(!mi_option_is_enabled(mi_option_eager_commit) || *commit);
mi_bitmap_unclaim(&region->reset, 1, blocks, bit_idx);
bool reset_zero;
_mi_mem_unreset(p, blocks * MI_SEGMENT_SIZE, &reset_zero, tld);
if (reset_zero) *is_zero = true;
}
mi_assert_internal(!mi_bitmap_is_any_claimed(&region->reset, 1, blocks, bit_idx));
#if (MI_DEBUG>=2)
if (*commit) { ((uint8_t*)p)[0] = 0; }
#endif
// and return the allocation
mi_assert_internal(p != NULL);
@ -325,7 +346,9 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l
void* p = mi_region_try_alloc(blocks, commit, large, is_zero, memid, tld);
mi_assert_internal(p == NULL || (uintptr_t)p % alignment == 0);
if (p != NULL) {
#if (MI_DEBUG>=2)
if (*commit) { ((uint8_t*)p)[0] = 0; }
#endif
return p;
}
_mi_warning_message("unable to allocate from region: size %zu\n", size);
@ -346,56 +369,56 @@ Free
-----------------------------------------------------------------------------*/
// Free previously allocated memory with a given id.
void _mi_mem_free(void* p, size_t size, size_t id, mi_os_tld_t* tld) {
void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_reset, mi_os_tld_t* tld) {
mi_assert_internal(size > 0 && tld != NULL);
if (p==NULL) return;
if (size==0) return;
size = _mi_align_up(size, _mi_os_page_size());
size_t arena_memid = 0;
mi_bitmap_index_t bit_idx;
mem_region_t* region;
if (mi_memid_indices(id,&region,&bit_idx,&arena_memid)) {
if (mi_memid_is_arena(id,&region,&bit_idx,&arena_memid)) {
// was a direct arena allocation, pass through
_mi_arena_free(p, size, arena_memid, tld->stats);
}
else {
// allocated in a region
mi_assert_internal(size <= MI_REGION_MAX_OBJ_SIZE); if (size > MI_REGION_MAX_OBJ_SIZE) return;
// we can align the size up to page size (as we allocate that way too)
// this ensures we fully commit/decommit/reset
size = _mi_align_up(size, _mi_os_page_size());
const size_t blocks = mi_region_block_count(size);
mi_region_info_t info = mi_atomic_read(&region->info);
bool is_large;
bool is_committed;
void* start = mi_region_info_read(info, &is_large, &is_committed);
mi_assert_internal(start != NULL);
void* blocks_start = (uint8_t*)start + (bit_idx * MI_SEGMENT_SIZE);
mi_assert_internal(blocks + bit_idx <= MI_BITMAP_FIELD_BITS);
mi_region_info_t info;
info.value = mi_atomic_read(&region->info);
mi_assert_internal(info.value != 0);
void* blocks_start = mi_region_blocks_start(region, bit_idx);
mi_assert_internal(blocks_start == p); // not a pointer in our area?
mi_assert_internal(bit_idx + blocks <= MI_BITMAP_FIELD_BITS);
if (blocks_start != p || bit_idx + blocks > MI_BITMAP_FIELD_BITS) return; // or `abort`?
// decommit (or reset) the blocks to reduce the working set.
// TODO: implement delayed decommit/reset as these calls are too expensive
// if the memory is reused soon.
// reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large
if (!is_large &&
mi_option_is_enabled(mi_option_segment_reset) &&
mi_option_is_enabled(mi_option_eager_commit)) // cannot reset halfway committed segments, use `option_page_reset` instead
{
// note: don't use `_mi_mem_reset` as it is shared with other threads!
_mi_os_reset(p, size, tld->stats); // TODO: maintain reset bits to unreset
}
if (!is_committed) {
// adjust commit statistics as we commit again when re-using the same slot
_mi_stat_decrease(&tld->stats->committed, mi_good_commit_size(size));
// committed?
if (full_commit && (size % MI_SEGMENT_SIZE) == 0) {
mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, NULL);
}
// TODO: should we free empty regions? currently only done _mi_mem_collect.
// this frees up virtual address space which might be useful on 32-bit systems?
if (any_reset) {
// set the is_reset bits if any pages were reset
mi_bitmap_claim(&region->reset, 1, blocks, bit_idx, NULL);
}
// reset the blocks to reduce the working set.
if (!info.is_large && mi_option_is_enabled(mi_option_segment_reset) &&
mi_option_is_enabled(mi_option_eager_commit)) // cannot reset halfway committed segments, use only `option_page_reset` instead
{
bool any_unreset;
mi_bitmap_claim(&region->reset, 1, blocks, bit_idx, &any_unreset);
if (any_unreset) {
_mi_mem_reset(p, blocks * MI_SEGMENT_SIZE, tld);
}
}
// and unclaim
mi_bitmap_unclaim(&region->in_use, 1, blocks, bit_idx);
bool all_unclaimed = mi_bitmap_unclaim(&region->in_use, 1, blocks, bit_idx);
mi_assert_internal(all_unclaimed); UNUSED(all_unclaimed);
}
}
@ -416,13 +439,14 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
} while(m == 0 && !mi_atomic_cas_weak(&region->in_use, MI_BITMAP_FIELD_FULL, 0 ));
if (m == 0) {
// on success, free the whole region
bool is_eager_committed;
void* start = mi_region_info_read(mi_atomic_read(&regions[i].info), NULL, &is_eager_committed);
if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) {
_mi_arena_free(start, MI_REGION_SIZE, region->arena_memid, tld->stats);
void* start = mi_atomic_read_ptr(&regions[i].start);
size_t arena_memid = mi_atomic_read_relaxed(&regions[i].arena_memid);
memset(&regions[i], 0, sizeof(mem_region_t));
// and release the whole region
mi_atomic_write(&region->info, 0);
if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) {
_mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats);
}
// and release
mi_atomic_write(&region->info,0);
}
}
}
@ -432,6 +456,7 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
/* ----------------------------------------------------------------------------
Other
-----------------------------------------------------------------------------*/
bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld) {
return _mi_os_reset(p, size, tld->stats);
}

5
src/options.c
View File

@ -65,10 +65,11 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
{ 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread
{ 1, UNINIT, MI_OPTION(page_reset) }, // reset pages on free
{ 0, UNINIT, MI_OPTION(page_reset) }, // reset pages on free
{ 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit)
{ 1, UNINIT, MI_OPTION(reset_decommits) }, // reset decommits memory
{ 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
{ 1, UNINIT, MI_OPTION(reset_decommits) }, // reset uses decommit/commit
{ 500,UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
{ 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output

204
src/os.c
View File

@ -77,11 +77,11 @@ static bool use_large_os_page(size_t size, size_t alignment) {
// round to a good OS allocation size (bounded by max 12.5% waste)
size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size;
if (size < 512 * KiB) align_size = _mi_os_page_size();
else if (size < 2 * MiB) align_size = 64 * KiB;
else if (size < 8 * MiB) align_size = 256 * KiB;
else if (size < 32 * MiB) align_size = 1 * MiB;
else align_size = 4 * MiB;
if (size < 512*KiB) align_size = _mi_os_page_size();
else if (size < 2*MiB) align_size = 64*KiB;
else if (size < 8*MiB) align_size = 256*KiB;
else if (size < 32*MiB) align_size = 1*MiB;
else align_size = 4*MiB;
if (size >= (SIZE_MAX - align_size)) return size; // possible overflow?
return _mi_align_up(size, align_size);
}
@ -92,8 +92,8 @@ size_t _mi_os_good_alloc_size(size_t size) {
// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
// We hide MEM_EXTENDED_PARAMETER to compile with older SDK's.
#include <winternl.h>
typedef PVOID(__stdcall* PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
typedef NTSTATUS(__stdcall* PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ PVOID, ULONG);
typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ PVOID, ULONG);
static PVirtualAlloc2 pVirtualAlloc2 = NULL;
static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
@ -129,7 +129,7 @@ static bool mi_win_enable_large_os_pages()
if (err == 0) err = GetLastError();
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
}
return (ok != 0);
return (ok!=0);
}
void _mi_os_init(void) {
@ -144,7 +144,7 @@ void _mi_os_init(void) {
if (hDll != NULL) {
// use VirtualAlloc2FromApp if possible as it is available to Windows store apps
pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2 == NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
FreeLibrary(hDll);
}
hDll = LoadLibrary(TEXT("ntdll.dll"));
@ -170,7 +170,7 @@ void _mi_os_init() {
os_alloc_granularity = os_page_size;
}
if (mi_option_is_enabled(mi_option_large_os_pages)) {
large_os_page_size = 2 * MiB;
large_os_page_size = 2*MiB;
}
}
#endif
@ -210,7 +210,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
#if (MI_INTPTR_SIZE >= 8)
// on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations
void* hint;
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) {
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) {
return VirtualAlloc(hint, size, flags, PAGE_READWRITE);
}
#endif
@ -233,7 +233,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
void* p = NULL;
if ((large_only || use_large_os_page(size, try_alignment))
&& allow_large && (flags & MEM_COMMIT) != 0 && (flags & MEM_RESERVE) != 0) {
&& allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) {
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
if (!large_only && try_ok > 0) {
// if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
@ -247,12 +247,12 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
if (large_only) return p;
// fall back to non-large page allocation on error (`p == NULL`).
if (p == NULL) {
mi_atomic_write(&large_page_try_ok, 10); // on error, don't try again for the next N allocations
mi_atomic_write(&large_page_try_ok,10); // on error, don't try again for the next N allocations
}
}
}
if (p == NULL) {
*is_large = ((flags & MEM_LARGE_PAGES) != 0);
*is_large = ((flags&MEM_LARGE_PAGES) != 0);
p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
}
if (p == NULL) {
@ -264,8 +264,8 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
#elif defined(__wasi__)
static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size();
uintptr_t aligned_base = _mi_align_up(base, (uintptr_t)try_alignment);
size_t alloc_size = _mi_align_up(aligned_base - base + size, _mi_os_page_size());
uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment);
size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size());
mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0);
if (alloc_size < size) return NULL;
if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) {
@ -278,50 +278,50 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
#define MI_OS_USE_MMAP
static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
void* p = NULL;
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
// on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations
void* hint;
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) {
p = mmap(hint, size, protect_flags, flags, fd, 0);
if (p == MAP_FAILED) p = NULL; // fall back to regular mmap
p = mmap(hint,size,protect_flags,flags,fd,0);
if (p==MAP_FAILED) p = NULL; // fall back to regular mmap
}
#else
#else
UNUSED(try_alignment);
#endif
if (p == NULL) {
p = mmap(addr, size, protect_flags, flags, fd, 0);
if (p == MAP_FAILED) p = NULL;
#endif
if (p==NULL) {
p = mmap(addr,size,protect_flags,flags,fd,0);
if (p==MAP_FAILED) p = NULL;
}
return p;
}
static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
void* p = NULL;
#if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON
#endif
#if !defined(MAP_NORESERVE)
#define MAP_NORESERVE 0
#endif
#if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON
#endif
#if !defined(MAP_NORESERVE)
#define MAP_NORESERVE 0
#endif
int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
int fd = -1;
#if defined(MAP_ALIGNED) // BSD
#if defined(MAP_ALIGNED) // BSD
if (try_alignment > 0) {
size_t n = _mi_bsr(try_alignment);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
flags |= MAP_ALIGNED(n);
}
}
#endif
#if defined(PROT_MAX)
#endif
#if defined(PROT_MAX)
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
#endif
#if defined(VM_MAKE_TAG)
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
#endif
#if defined(VM_MAKE_TAG)
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
int os_tag = (int)mi_option_get(mi_option_os_tag);
if (os_tag < 100 || os_tag > 255) os_tag = 100;
fd = VM_MAKE_TAG(os_tag);
#endif
#endif
if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) {
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
@ -335,39 +335,39 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
else {
int lflags = flags;
int lfd = fd;
#ifdef MAP_ALIGNED_SUPER
#ifdef MAP_ALIGNED_SUPER
lflags |= MAP_ALIGNED_SUPER;
#endif
#ifdef MAP_HUGETLB
#endif
#ifdef MAP_HUGETLB
lflags |= MAP_HUGETLB;
#endif
#ifdef MAP_HUGE_1GB
#endif
#ifdef MAP_HUGE_1GB
static bool mi_huge_pages_available = true;
if ((size % GiB) == 0 && mi_huge_pages_available) {
lflags |= MAP_HUGE_1GB;
}
else
#endif
#endif
{
#ifdef MAP_HUGE_2MB
#ifdef MAP_HUGE_2MB
lflags |= MAP_HUGE_2MB;
#endif
#endif
}
#ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
#ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
#endif
#endif
if (large_only || lflags != flags) {
// try large OS page allocation
*is_large = true;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB
#ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) {
mi_huge_pages_available = false; // don't try huge 1GiB pages again
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno);
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
}
#endif
#endif
if (large_only) return p;
if (p == NULL) {
mi_atomic_write(&large_page_try_ok, 10); // on error, don't try again for the next N allocations
@ -378,7 +378,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
if (p == NULL) {
*is_large = false;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd);
#if defined(MADV_HUGEPAGE)
#if defined(MADV_HUGEPAGE)
// Many Linux systems don't allow MAP_HUGETLB but they support instead
// transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE
// though since properly aligned allocations will already use large pages if available
@ -390,7 +390,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
*is_large = true; // possibly
};
}
#endif
#endif
}
return p;
}
@ -404,18 +404,18 @@ static volatile _Atomic(intptr_t) aligned_base;
// Return a 4MiB aligned address that is probably available
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL;
if ((size % MI_SEGMENT_SIZE) != 0) return NULL;
if ((size%MI_SEGMENT_SIZE) != 0) return NULL;
intptr_t hint = mi_atomic_add(&aligned_base, size);
if (hint == 0 || hint > ((intptr_t)30 << 40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages)
if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages)
intptr_t init = ((intptr_t)4 << 40); // start at 4TiB area
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_os_get_aligned_hint ^ hint);
init = init + (MI_SEGMENT_SIZE * ((r >> 17) & 0xFFFF)); // (randomly 0-64k)*4MiB == 0 to 256GiB
#endif
init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFF)); // (randomly 0-64k)*4MiB == 0 to 256GiB
#endif
mi_atomic_cas_strong(mi_atomic_cast(uintptr_t, &aligned_base), init, hint + size);
hint = mi_atomic_add(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all
}
if (hint % try_alignment != 0) return NULL;
if (hint%try_alignment != 0) return NULL;
return (void*)hint;
}
#else
@ -444,17 +444,17 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo
}
*/
#if defined(_WIN32)
int flags = MEM_RESERVE;
if (commit) flags |= MEM_COMMIT;
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
#elif defined(__wasi__)
*is_large = false;
p = mi_wasm_heap_grow(size, try_alignment);
#else
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
#endif
#if defined(_WIN32)
int flags = MEM_RESERVE;
if (commit) flags |= MEM_COMMIT;
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
#elif defined(__wasi__)
*is_large = false;
p = mi_wasm_heap_grow(size, try_alignment);
#else
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
#endif
mi_stat_counter_increase(stats->mmap_calls, 1);
if (p != NULL) {
_mi_stat_increase(&stats->reserved, size);
@ -564,7 +564,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* lar
allow_large = *large;
*large = false;
}
return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large != NULL ? large : &allow_large), tld->stats);
return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large!=NULL?large:&allow_large), tld->stats);
}
@ -616,7 +616,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
_mi_stat_decrease(&stats->committed, csize);
}
#if defined(_WIN32)
#if defined(_WIN32)
if (commit) {
// if the memory was already committed, the call succeeds but it is not zero'd
// *is_zero = true;
@ -627,9 +627,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
BOOL ok = VirtualFree(start, csize, MEM_DECOMMIT);
err = (ok ? 0 : GetLastError());
}
#elif defined(__wasi__)
#elif defined(__wasi__)
// WebAssembly guests can't control memory protection
#elif defined(MAP_FIXED)
#elif defined(MAP_FIXED)
if (!commit) {
// use mmap with MAP_FIXED to discard the existing memory (and reduce commit charge)
void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
@ -640,10 +640,10 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
if (err != 0) { err = errno; }
}
#else
#else
err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
if (err != 0) { err = errno; }
#endif
#endif
if (err != 0) {
_mi_warning_message("%s error: start: 0x%p, csize: 0x%x, err: %i\n", commit ? "commit" : "decommit", start, csize, err);
}
@ -674,24 +674,24 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
if (reset) _mi_stat_increase(&stats->reset, csize);
else _mi_stat_decrease(&stats->reset, csize);
else _mi_stat_decrease(&stats->reset, csize);
if (!reset) return true; // nothing to do on unreset!
#if (MI_DEBUG>1)
if (MI_SECURE == 0) {
#if (MI_DEBUG>1)
if (MI_SECURE==0) {
memset(start, 0, csize); // pretend it is eagerly reset
}
#endif
#endif
#if defined(_WIN32)
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == start);
#if 1
#if 1
if (p == start && start != NULL) {
VirtualUnlock(start, csize); // VirtualUnlock after MEM_RESET removes the memory from the working set
VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set
}
#endif
#endif
if (p != start) return false;
#else
#if defined(MADV_FREE)
@ -748,7 +748,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
if (csize == 0) return false;
/*
if (_mi_os_is_huge_reserved(addr)) {
_mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
_mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
}
*/
int err = 0;
@ -780,7 +780,7 @@ bool _mi_os_unprotect(void* addr, size_t size) {
bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
// page align conservatively within the range
mi_assert_internal(oldsize > newsize&& p != NULL);
mi_assert_internal(oldsize > newsize && p != NULL);
if (oldsize < newsize || p == NULL) return false;
if (oldsize == newsize) return true;
@ -808,20 +808,20 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`)
#if defined(WIN32) && (MI_INTPTR_SIZE >= 8)
static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
{
mi_assert_internal(size % GiB == 0);
mi_assert_internal(size%GiB == 0);
mi_assert_internal(addr != NULL);
const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;
mi_win_enable_large_os_pages();
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
MEM_EXTENDED_PARAMETER params[3] = { {0,0},{0,0},{0,0} };
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
static bool mi_huge_pages_available = true;
if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) {
#ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
#define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE (0x10)
#endif
#ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
#define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE (0x10)
#endif
params[0].Type = 5; // == MemExtendedParameterAttributeFlags;
params[0].ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
ULONG param_count = 1;
@ -848,7 +848,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
params[0].ULong = (unsigned)numa_node;
return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
}
#endif
#endif
// otherwise use regular virtual alloc on older windows
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
@ -869,16 +869,16 @@ static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, cons
}
#endif
static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) {
mi_assert_internal(size % GiB == 0);
mi_assert_internal(size%GiB == 0);
bool is_large = true;
void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large);
if (p == NULL) return NULL;
if (numa_node >= 0 && numa_node < 8 * MI_INTPTR_SIZE) { // at most 64 nodes
if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes
uintptr_t numa_mask = (1UL << numa_node);
// TODO: does `mbind` work correctly for huge OS pages? should we
// use `set_mempolicy` before calling mmap instead?
// see: <https://lkml.org/lkml/2017/2/9/875>
long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8 * MI_INTPTR_SIZE, 0);
long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
if (err != 0) {
_mi_warning_message("failed to bind huge (1GiB) pages to NUMA node %d: %s\n", numa_node, strerror(errno));
}
@ -910,7 +910,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
start = ((uintptr_t)32 << 40); // 32TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_os_claim_huge_pages);
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r >> 17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB
#endif
}
end = start + size;
@ -963,8 +963,8 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
if (max_msecs > 0) {
mi_msecs_t elapsed = _mi_clock_end(start_t);
if (page >= 1) {
mi_msecs_t estimate = ((elapsed / (page + 1)) * pages);
if (estimate > 2 * max_msecs) { // seems like we are going to timeout, break
mi_msecs_t estimate = ((elapsed / (page+1)) * pages);
if (estimate > 2*max_msecs) { // seems like we are going to timeout, break
elapsed = max_msecs + 1;
}
}
@ -974,7 +974,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
}
}
}
mi_assert_internal(page * MI_HUGE_OS_PAGE_SIZE <= size);
mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size);
if (pages_reserved != NULL) *pages_reserved = page;
if (psize != NULL) *psize = page * MI_HUGE_OS_PAGE_SIZE;
return (page == 0 ? NULL : start);
@ -983,7 +983,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// free every huge page in a range individually (as we allocated per page)
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
if (p == NULL || size == 0) return;
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
_mi_os_free(base, MI_HUGE_OS_PAGE_SIZE, stats);
@ -999,7 +999,7 @@ static size_t mi_os_numa_nodex() {
PROCESSOR_NUMBER pnum;
USHORT numa_node = 0;
GetCurrentProcessorNumberEx(&pnum);
GetNumaProcessorNodeEx(&pnum, &numa_node);
GetNumaProcessorNodeEx(&pnum,&numa_node);
return numa_node;
}
@ -1026,12 +1026,12 @@ static size_t mi_os_numa_nodex(void) {
static size_t mi_os_numa_node_countx(void) {
char buf[128];
unsigned node = 0;
for (node = 0; node < 256; node++) {
for(node = 0; node < 256; node++) {
// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
if (access(buf, R_OK) != 0) break;
if (access(buf,R_OK) != 0) break;
}
return (node + 1);
return (node+1);
}
#else
static size_t mi_os_numa_nodex(void) {
@ -1058,7 +1058,7 @@ size_t _mi_os_numa_node_count_get(void) {
int _mi_os_numa_node_get(mi_os_tld_t* tld) {
UNUSED(tld);
size_t numa_count = _mi_os_numa_node_count();
if (numa_count <= 1) return 0; // optimize on single numa node systems: always node 0
if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0
// never more than the node count and >= 0
size_t numa_node = mi_os_numa_nodex();
if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }

7
src/page.c
View File

@ -75,7 +75,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
mi_segment_t* segment = _mi_page_segment(page);
uint8_t* start = _mi_page_start(segment,page,NULL);
mi_assert_internal(start == _mi_segment_page_start(segment,page,page->block_size,NULL));
mi_assert_internal(start == _mi_segment_page_start(segment,page,page->block_size,NULL,NULL));
//mi_assert_internal(start + page->capacity*page->block_size == page->top);
mi_assert_internal(mi_page_list_is_valid(page,page->free));
@ -229,6 +229,7 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {
mi_assert_expensive(mi_page_is_valid_init(page));
mi_assert_internal(page->heap == NULL);
mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
mi_assert_internal(!page->is_reset);
_mi_page_free_collect(page,false);
mi_page_queue_t* pq = mi_page_queue(heap, page->block_size);
mi_page_queue_push(heap, pq, page);
@ -342,7 +343,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(page->heap != NULL);
#if MI_DEBUG > 1
mi_heap_t* pheap = (mi_heap_t*)mi_atomic_read_ptr(mi_atomic_cast(void*, &page->heap));
#endif
@ -597,7 +598,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert_internal(block_size > 0);
// set fields
size_t page_size;
_mi_segment_page_start(segment, page, block_size, &page_size);
_mi_segment_page_start(segment, page, block_size, &page_size, NULL);
page->block_size = block_size;
mi_assert_internal(page_size / block_size < (1L<<16));
page->reserved = (uint16_t)(page_size / block_size);

264
src/segment.c
View File

@ -13,6 +13,8 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_PAGE_HUGE_ALIGN (256*1024)
static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size);
/* -----------------------------------------------------------
Segment allocation
We allocate pages inside big OS allocated "segments"
@ -40,7 +42,6 @@ terms of the MIT license. A copy of the license can be found in the file
Queue of segments containing free pages
----------------------------------------------------------- */
#if (MI_DEBUG>=3)
static bool mi_segment_queue_contains(const mi_segment_queue_t* queue, mi_segment_t* segment) {
mi_assert_internal(segment != NULL);
@ -143,31 +144,50 @@ static bool mi_segment_is_valid(mi_segment_t* segment) {
}
#endif
/* -----------------------------------------------------------
Page reset
----------------------------------------------------------- */
static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld) {
if (!mi_option_is_enabled(mi_option_page_reset)) return;
if (segment->mem_is_fixed || page->segment_in_use || page->is_reset) return;
size_t psize;
void* start = mi_segment_raw_page_start(segment, page, &psize);
page->is_reset = true;
mi_assert_internal(size <= psize);
_mi_mem_reset(start, ((size == 0 || size > psize) ? psize : size), tld->os);
}
static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld)
{
mi_assert_internal(page->is_reset);
mi_assert_internal(!segment->mem_is_fixed);
page->is_reset = false;
size_t psize;
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
bool is_zero = false;
_mi_mem_unreset(start, ((size == 0 || size > psize) ? psize : size), &is_zero, tld->os);
if (is_zero) page->is_zero_init = true;
}
/* -----------------------------------------------------------
Segment size calculations
----------------------------------------------------------- */
// Start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set)
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size)
{
// Raw start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set)
// The raw start is not taking aligned block allocation into consideration.
static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) {
size_t psize = (segment->page_kind == MI_PAGE_HUGE ? segment->segment_size : (size_t)1 << segment->page_shift);
uint8_t* p = (uint8_t*)segment + page->segment_idx*psize;
uint8_t* p = (uint8_t*)segment + page->segment_idx * psize;
if (page->segment_idx == 0) {
// the first page starts after the segment info (and possible guard page)
p += segment->segment_info_size;
p += segment->segment_info_size;
psize -= segment->segment_info_size;
// for small and medium objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore)
if (block_size > 0 && segment->page_kind <= MI_PAGE_MEDIUM) {
size_t adjust = block_size - ((uintptr_t)p % block_size);
if (adjust < block_size) {
p += adjust;
psize -= adjust;
}
mi_assert_internal((uintptr_t)p % block_size == 0);
}
}
if (MI_SECURE > 1 || (MI_SECURE == 1 && page->segment_idx == segment->capacity - 1)) {
// secure == 1: the last page has an os guard page at the end
// secure > 1: every page has an os guard page
@ -175,19 +195,36 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
}
if (page_size != NULL) *page_size = psize;
mi_assert_internal(_mi_ptr_page(p) == page);
mi_assert_internal(page->block_size == 0 || _mi_ptr_page(p) == page);
mi_assert_internal(_mi_ptr_segment(p) == segment);
return p;
}
static size_t mi_segment_size(size_t capacity, size_t required, size_t* pre_size, size_t* info_size) {
/*
if (mi_option_is_enabled(mi_option_secure)) {
// always reserve maximally so the protection falls on
// the same address area, as we need to reuse them from the caches interchangably.
capacity = MI_SMALL_PAGES_PER_SEGMENT;
// Start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set)
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size, size_t* pre_size)
{
size_t psize;
uint8_t* p = mi_segment_raw_page_start(segment, page, &psize);
if (pre_size != NULL) *pre_size = 0;
if (page->segment_idx == 0 && block_size > 0 && segment->page_kind <= MI_PAGE_MEDIUM) {
// for small and medium objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore)
size_t adjust = block_size - ((uintptr_t)p % block_size);
if (adjust < block_size) {
p += adjust;
psize -= adjust;
if (pre_size != NULL) *pre_size = adjust;
}
mi_assert_internal((uintptr_t)p % block_size == 0);
}
*/
if (page_size != NULL) *page_size = psize;
mi_assert_internal(page->block_size==0 || _mi_ptr_page(p) == page);
mi_assert_internal(_mi_ptr_segment(p) == segment);
return p;
}
static size_t mi_segment_size(size_t capacity, size_t required, size_t* pre_size, size_t* info_size)
{
const size_t minsize = sizeof(mi_segment_t) + ((capacity - 1) * sizeof(mi_page_t)) + 16 /* padding */;
size_t guardsize = 0;
size_t isize = 0;
@ -234,7 +271,15 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
mi_assert_internal(!segment->mem_is_fixed);
_mi_mem_unprotect(segment, segment->segment_size); // ensure no more guard pages are set
}
_mi_mem_free(segment, segment_size, segment->memid, tld->os);
bool fully_committed = true;
bool any_reset = false;
for (size_t i = 0; i < segment->capacity; i++) {
const mi_page_t* page = &segment->pages[i];
if (!page->is_committed) fully_committed = false;
if (page->is_reset) any_reset = true;
}
_mi_mem_free(segment, segment_size, segment->memid, fully_committed, any_reset, tld->os);
}
@ -275,7 +320,7 @@ static bool mi_segment_cache_full(mi_segments_tld_t* tld)
static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(!mi_segment_is_in_free_queue(segment, tld));
mi_assert_internal(segment->next == NULL);
mi_assert_internal(segment->next == NULL);
if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) {
return false;
}
@ -328,31 +373,31 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay));
bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit);
bool commit = eager || (page_kind >= MI_PAGE_LARGE);
bool protection_still_good = false;
bool pages_still_good = false;
bool is_zero = false;
// Try to get it from our thread local cache first
mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld);
mi_segment_t* segment = NULL; // mi_segment_cache_pop(segment_size, tld);
if (segment != NULL) {
if (MI_SECURE!=0) {
mi_assert_internal(!segment->mem_is_fixed);
if (segment->page_kind != page_kind) {
if (page_kind <= MI_PAGE_MEDIUM && segment->page_kind == page_kind && segment->segment_size == segment_size) {
pages_still_good = true;
}
else
{
// different page kinds; unreset any reset pages, and unprotect
// TODO: optimize cache pop to return fitting pages if possible?
for (size_t i = 0; i < segment->capacity; i++) {
mi_page_t* page = &segment->pages[i];
if (page->is_reset) {
mi_page_unreset(segment, page, 0, tld); // todo: only unreset the part that was reset? (instead of the full page)
}
}
if (MI_SECURE!=0) {
mi_assert_internal(!segment->mem_is_fixed);
// TODO: should we unprotect per page? (with is_protected flag?)
_mi_mem_unprotect(segment, segment->segment_size); // reset protection if the page kind differs
}
else {
protection_still_good = true; // otherwise, the guard pages are still in place
}
}
if (!segment->mem_is_committed && page_kind > MI_PAGE_MEDIUM) {
mi_assert_internal(!segment->mem_is_fixed);
_mi_mem_commit(segment, segment->segment_size, &is_zero, tld->os);
segment->mem_is_committed = true;
}
if (!segment->mem_is_fixed && mi_option_is_enabled(mi_option_page_reset)) {
bool reset_zero = false;
_mi_mem_unreset(segment, segment->segment_size, &reset_zero, tld->os);
if (reset_zero) is_zero = true;
}
}
}
else {
// Allocate the segment from the OS
@ -373,27 +418,42 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
}
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
// zero the segment info (but not the `mem` fields)
ptrdiff_t ofs = offsetof(mi_segment_t,next);
memset((uint8_t*)segment + ofs, 0, info_size - ofs);
// guard pages
if ((MI_SECURE != 0) && !protection_still_good) {
// in secure mode, we set up a protected page in between the segment info
// and the page data
mi_assert_internal( info_size == pre_size - _mi_os_page_size() && info_size % _mi_os_page_size() == 0);
_mi_mem_protect( (uint8_t*)segment + info_size, (pre_size - info_size) );
size_t os_page_size = _mi_os_page_size();
if (MI_SECURE <= 1) {
// and protect the last page too
_mi_mem_protect( (uint8_t*)segment + segment_size - os_page_size, os_page_size );
}
else {
// protect every page
for (size_t i = 0; i < capacity; i++) {
_mi_mem_protect( (uint8_t*)segment + (i+1)*page_size - os_page_size, os_page_size );
if (!pages_still_good) {
// guard pages
if (MI_SECURE != 0) {
// in secure mode, we set up a protected page in between the segment info
// and the page data
mi_assert_internal(info_size == pre_size - _mi_os_page_size() && info_size % _mi_os_page_size() == 0);
_mi_mem_protect((uint8_t*)segment + info_size, (pre_size - info_size));
const size_t os_page_size = _mi_os_page_size();
if (MI_SECURE <= 1) {
// and protect the last page too
_mi_mem_protect((uint8_t*)segment + segment_size - os_page_size, os_page_size);
}
else {
// protect every page
for (size_t i = 0; i < capacity; i++) {
_mi_mem_protect((uint8_t*)segment + (i+1)*page_size - os_page_size, os_page_size);
}
}
}
// zero the segment info (but not the `mem` fields)
ptrdiff_t ofs = offsetof(mi_segment_t, next);
memset((uint8_t*)segment + ofs, 0, info_size - ofs);
// initialize pages info
for (uint8_t i = 0; i < capacity; i++) {
segment->pages[i].segment_idx = i;
segment->pages[i].is_reset = false;
segment->pages[i].is_committed = commit;
segment->pages[i].is_zero_init = is_zero;
}
}
else {
// zero the segment info but not the pages info (and mem fields)
ptrdiff_t ofs = offsetof(mi_segment_t, next);
memset((uint8_t*)segment + ofs, 0, offsetof(mi_segment_t,pages) - ofs);
}
// initialize
@ -404,13 +464,8 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
segment->segment_info_size = pre_size;
segment->thread_id = _mi_thread_id();
segment->cookie = _mi_ptr_cookie(segment);
for (uint8_t i = 0; i < segment->capacity; i++) {
segment->pages[i].segment_idx = i;
segment->pages[i].is_reset = false;
segment->pages[i].is_committed = commit;
segment->pages[i].is_zero_init = is_zero;
}
_mi_stat_increase(&tld->stats->page_committed, segment->segment_info_size);
//fprintf(stderr,"mimalloc: alloc segment at %p\n", (void*)segment);
return segment;
}
@ -463,24 +518,22 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_segments_tld_t*
for (size_t i = 0; i < segment->capacity; i++) {
mi_page_t* page = &segment->pages[i];
if (!page->segment_in_use) {
if (page->is_reset || !page->is_committed) {
// set in-use before doing unreset to prevent delayed reset
page->segment_in_use = true;
segment->used++;
if (!page->is_committed) {
mi_assert_internal(!segment->mem_is_fixed);
mi_assert_internal(!page->is_reset);
size_t psize;
uint8_t* start = _mi_page_start(segment, page, &psize);
if (!page->is_committed) {
mi_assert_internal(!segment->mem_is_fixed);
page->is_committed = true;
bool is_zero = false;
_mi_mem_commit(start,psize,&is_zero,tld->os);
if (is_zero) page->is_zero_init = true;
}
if (page->is_reset) {
mi_assert_internal(!segment->mem_is_fixed);
page->is_reset = false;
bool is_zero = false;
_mi_mem_unreset(start, psize, &is_zero, tld->os);
if (is_zero) page->is_zero_init = true;
}
uint8_t* start = _mi_page_start(segment, page, &psize);
page->is_committed = true;
bool is_zero = false;
_mi_mem_commit(start,psize,&is_zero,tld->os);
if (is_zero) page->is_zero_init = true;
}
if (page->is_reset) {
mi_page_unreset(segment, page, 0, tld); // todo: only unreset the part that was reset?
}
return page;
}
}
@ -503,22 +556,21 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_seg
_mi_stat_decrease(&tld->stats->page_committed, inuse);
_mi_stat_decrease(&tld->stats->pages, 1);
// reset the page memory to reduce memory pressure?
if (!segment->mem_is_fixed && !page->is_reset && mi_option_is_enabled(mi_option_page_reset))
// && segment->page_kind <= MI_PAGE_MEDIUM) // to prevent partial overlapping resets
{
size_t psize;
uint8_t* start = _mi_page_start(segment, page, &psize);
page->is_reset = true;
_mi_mem_reset(start, psize, tld->os);
}
// calculate the used size from the raw (non-aligned) start of the page
size_t pre_size;
_mi_segment_page_start(segment, page, page->block_size, NULL, &pre_size);
size_t used_size = pre_size + (page->capacity * page->block_size);
// zero the page data, but not the segment fields
// zero the page data, but not the segment fields
page->is_zero_init = false;
ptrdiff_t ofs = offsetof(mi_page_t,capacity);
memset((uint8_t*)page + ofs, 0, sizeof(*page) - ofs);
page->segment_in_use = false;
segment->used--;
// reset the page memory to reduce memory pressure?
// note: must come after setting `segment_in_use` to false
mi_page_reset(segment, page, used_size, tld);
}
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)
@ -568,7 +620,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
// remove the segment from the free page queue if needed
mi_segment_remove_from_free_queue(segment,tld);
mi_assert_internal(segment->next == NULL && segment->prev == NULL);
// all pages in the segment are abandoned; add it to the abandoned list
_mi_stat_increase(&tld->stats->segments_abandoned, 1);
mi_segments_track_size(-((long)segment->segment_size), tld);
@ -628,6 +680,8 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
for (size_t i = 0; i < segment->capacity; i++) {
mi_page_t* page = &segment->pages[i];
if (page->segment_in_use) {
mi_assert_internal(!page->is_reset);
mi_assert_internal(page->is_committed);
segment->abandoned--;
mi_assert(page->next == NULL);
_mi_stat_decrease(&tld->stats->pages_abandoned, 1);
@ -636,7 +690,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
mi_segment_page_clear(segment,page,tld);
}
else {
// otherwise reclaim it
// otherwise reclaim it
_mi_page_reclaim(heap,page);
}
}
@ -666,8 +720,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
static mi_page_t* mi_segment_page_alloc_in(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(mi_segment_has_free(segment));
mi_page_t* page = mi_segment_find_free(segment, tld);
page->segment_in_use = true;
segment->used++;
mi_assert_internal(page->segment_in_use);
mi_assert_internal(segment->used <= segment->capacity);
if (segment->used == segment->capacity) {
// if no more free pages, remove from the queue
@ -685,7 +738,11 @@ static mi_page_t* mi_segment_page_alloc(mi_page_kind_t kind, size_t page_shift,
mi_segment_enqueue(free_queue, segment);
}
mi_assert_internal(free_queue->first != NULL);
return mi_segment_page_alloc_in(free_queue->first,tld);
mi_page_t* page = mi_segment_page_alloc_in(free_queue->first,tld);
#if MI_DEBUG>=2
_mi_segment_page_start(_mi_page_segment(page), page, sizeof(void*), NULL, NULL)[0] = 0;
#endif
return page;
}
static mi_page_t* mi_segment_small_page_alloc(mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
@ -706,6 +763,9 @@ static mi_page_t* mi_segment_large_page_alloc(mi_segments_tld_t* tld, mi_os_tld_
segment->used = 1;
mi_page_t* page = &segment->pages[0];
page->segment_in_use = true;
#if MI_DEBUG>=2
_mi_segment_page_start(segment, page, sizeof(void*), NULL, NULL)[0] = 0;
#endif
return page;
}
@ -717,7 +777,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld
segment->used = 1;
segment->thread_id = 0; // huge pages are immediately abandoned
mi_page_t* page = &segment->pages[0];
page->segment_in_use = true;
page->segment_in_use = true;
return page;
}