mirrors/mimalloc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge from dev

Browse Source
This commit is contained in:
daan 2020-01-15 18:03:37 -08:00
commit 7a1e86fc20
16 changed files with 447 additions and 406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/mimalloc-config-version.cmake
View File

@ -1,5 +1,5 @@
set(mi_version_major 1)
set(mi_version_minor 3)
set(mi_version_minor 4)
set(mi_version ${mi_version_major}.${mi_version_minor})
set(PACKAGE_VERSION ${mi_version})

49
include/mimalloc-internal.h
View File

@ -33,8 +33,8 @@ terms of the MIT license. A copy of the license can be found in the file
// "options.c"
void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message);
void _mi_fprintf(mi_output_fun* out, const char* fmt, ...);
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
void _mi_error_message(const char* fmt, ...);
void _mi_warning_message(const char* fmt, ...);
void _mi_verbose_message(const char* fmt, ...);
@ -308,7 +308,7 @@ static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const
// Quick page start for initialized pages
static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) {
const size_t bsize = page->block_size;
const size_t bsize = page->xblock_size;
mi_assert_internal(bsize > 0 && (bsize%sizeof(void*)) == 0);
return _mi_segment_page_start(segment, page, bsize, page_size, NULL);
}
@ -318,7 +318,40 @@ static inline mi_page_t* _mi_ptr_page(void* p) {
return _mi_segment_page_of(_mi_ptr_segment(p), p);
}
// Get the block size of a page (special cased for huge objects)
static inline size_t mi_page_block_size(const mi_page_t* page) {
const size_t bsize = page->xblock_size;
mi_assert_internal(bsize > 0);
if (mi_likely(bsize < MI_HUGE_BLOCK_SIZE)) {
return bsize;
}
else {
size_t psize;
_mi_segment_page_start(_mi_page_segment(page), page, bsize, &psize, NULL);
return psize;
}
}
// Thread free access
static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
return (mi_block_t*)(mi_atomic_read_relaxed(&page->xthread_free) & ~3);
}
static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) {
return (mi_delayed_t)(mi_atomic_read_relaxed(&page->xthread_free) & 3);
}
// Heap access
static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
return (mi_heap_t*)(mi_atomic_read_relaxed(&page->xheap));
}
static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {
mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING);
mi_atomic_write(&page->xheap,(uintptr_t)heap);
}
// Thread free flag helpers
static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {
return (mi_block_t*)(tf & ~0x03);
}
@ -338,7 +371,7 @@ static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t*
// are all blocks in a page freed?
static inline bool mi_page_all_free(const mi_page_t* page) {
mi_assert_internal(page != NULL);
return (page->used - page->thread_freed == 0);
return (page->used == 0);
}
// are there immediately available blocks
@ -349,8 +382,8 @@ static inline bool mi_page_immediate_available(const mi_page_t* page) {
// are there free blocks in this page?
static inline bool mi_page_has_free(mi_page_t* page) {
mi_assert_internal(page != NULL);
bool hasfree = (mi_page_immediate_available(page) || page->local_free != NULL || (mi_tf_block(page->thread_free) != NULL));
mi_assert_internal(hasfree || page->used - page->thread_freed == page->capacity);
bool hasfree = (mi_page_immediate_available(page) || page->local_free != NULL || (mi_page_thread_free(page) != NULL));
mi_assert_internal(hasfree || page->used == page->capacity);
return hasfree;
}
@ -364,7 +397,7 @@ static inline bool mi_page_all_used(mi_page_t* page) {
static inline bool mi_page_mostly_used(const mi_page_t* page) {
if (page==NULL) return true;
uint16_t frac = page->reserved / 8U;
return (page->reserved - page->used + page->thread_freed <= frac);
return (page->reserved - page->used <= frac);
}
static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {
@ -467,7 +500,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t*
// check for free list corruption: is `next` at least in the same page?
// TODO: check if `next` is `page->block_size` aligned?
if (mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next))) {
_mi_fatal_error("corrupted free list entry of size %zub at %p: value 0x%zx\n", page->block_size, block, (uintptr_t)next);
_mi_fatal_error("corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
next = NULL;
}
return next;

57
include/mimalloc-types.h
View File

@ -124,6 +124,9 @@ terms of the MIT license. A copy of the license can be found in the file
#error "define more bins"
#endif
// Used as a special value to encode block sizes in 32 bits.
#define MI_HUGE_BLOCK_SIZE ((uint32_t)MI_HUGE_OBJ_SIZE_MAX)
// The free lists use encoded next fields
// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
typedef uintptr_t mi_encoded_t;
@ -136,10 +139,10 @@ typedef struct mi_block_s {
// The delayed flags are used for efficient multi-threaded free-ing
typedef enum mi_delayed_e {
MI_NO_DELAYED_FREE = 0,
MI_USE_DELAYED_FREE = 1,
MI_DELAYED_FREEING = 2,
MI_NEVER_DELAYED_FREE = 3
MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list
MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap
MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list
MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim
} mi_delayed_t;
@ -167,14 +170,28 @@ typedef uintptr_t mi_thread_free_t;
// implement a monotonic heartbeat. The `thread_free` list is needed for
// avoiding atomic operations in the common case.
//
// `used - thread_freed` == actual blocks that are in use (alive)
// `used - thread_freed + |free| + |local_free| == capacity`
//
// note: we don't count `freed` (as |free|) instead of `used` to reduce
// the number of memory accesses in the `mi_page_all_free` function(s).
// note: the funny layout here is due to:
// - access is optimized for `mi_free` and `mi_page_alloc`
// - using `uint16_t` does not seem to slow things down
// `used - |thread_free|` == actual blocks that are in use (alive)
// `used - |thread_free| + |free| + |local_free| == capacity`
//
// We don't count `freed` (as |free|) but use `used` to reduce
// the number of memory accesses in the `mi_page_all_free` function(s).
//
// Notes:
// - Access is optimized for `mi_free` and `mi_page_alloc` (in `alloc.c`)
// - Using `uint16_t` does not seem to slow things down
// - The size is 8 words on 64-bit which helps the page index calculations
// (and 10 words on 32-bit, and encoded free lists add 2 words. Sizes 10
// and 12 are still good for address calculation)
// - To limit the structure size, the `xblock_size` is 32-bits only; for
// blocks > MI_HUGE_BLOCK_SIZE the size is determined from the segment page size
// - `thread_free` uses the bottom bits as a delayed-free flags to optimize
// concurrent frees where only the first concurrent free adds to the owning
// heap `thread_delayed_free` list (see `alloc.c:mi_free_block_mt`).
// The invariant is that no-delayed-free is only set if there is
// at least one block that will be added, or as already been added, to
// the owning heap `thread_delayed_free` list. This guarantees that pages
// will be freed correctly even if only other threads free blocks.
typedef struct mi_page_s {
// "owned" by the segment
uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
@ -194,23 +211,15 @@ typedef struct mi_page_s {
#ifdef MI_ENCODE_FREELIST
uintptr_t key[2]; // two random keys to encode the free lists (see `_mi_block_next`)
#endif
size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`)
uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`)
uint32_t xblock_size; // size available in each block (always `>0`)
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
volatile _Atomic(uintptr_t) thread_freed; // at least this number of blocks are in `thread_free`
volatile _Atomic(mi_thread_free_t) thread_free; // list of deferred free blocks freed by other threads
// less accessed info
size_t block_size; // size available in each block (always `>0`)
mi_heap_t* heap; // the owning heap
volatile _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads
volatile _Atomic(uintptr_t) xheap;
struct mi_page_s* next; // next page owned by this thread with the same `block_size`
struct mi_page_s* prev; // previous page owned by this thread with the same `block_size`
// improve page index calculation
// without padding: 10 words on 64-bit, 11 on 32-bit. Secure adds two words
#if (MI_INTPTR_SIZE==4)
void* padding[1]; // 12/14 words on 32-bit plain
#endif
} mi_page_t;

15
include/mimalloc.h
View File

@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MIMALLOC_H
#define MIMALLOC_H
#define MI_MALLOC_VERSION 130 // major + 2 digits minor
#define MI_MALLOC_VERSION 140 // major + 2 digits minor
// ------------------------------------------------------
// Compiler specific attributes
@ -108,22 +108,23 @@ mi_decl_export mi_decl_allocator void* mi_reallocf(void* p, size_t newsize)
mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat);
mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free) mi_attr_noexcept;
typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;
typedef void (mi_output_fun)(const char* msg);
mi_decl_export void mi_register_output(mi_output_fun* out) mi_attr_noexcept;
typedef void (mi_output_fun)(const char* msg, void* arg);
mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;
mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
mi_decl_export int mi_version(void) mi_attr_noexcept;
mi_decl_export void mi_stats_reset(void) mi_attr_noexcept;
mi_decl_export void mi_stats_merge(void) mi_attr_noexcept;
mi_decl_export void mi_stats_print(mi_output_fun* out) mi_attr_noexcept;
mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL
mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
mi_decl_export void mi_process_init(void) mi_attr_noexcept;
mi_decl_export void mi_thread_init(void) mi_attr_noexcept;
mi_decl_export void mi_thread_done(void) mi_attr_noexcept;
mi_decl_export void mi_thread_stats_print(mi_output_fun* out) mi_attr_noexcept;
mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
// -------------------------------------------------------------------------------------

3
readme.md
View File

@ -56,6 +56,9 @@ Enjoy!
### Releases
* 2020-01-15, `v1.3.0`: stable release 1.3: bug fixes, improved randomness and stronger
free list encoding in secure mode.
* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). Improved dynamic overriding on Windows.
* 2019-10-07, `v1.1.0`: stable release 1.1.
* 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support.

131
src/alloc.c
View File

@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file
// Fast allocation in a page: just pop from the free list.
// Fall back to generic allocation only if the list is empty.
extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept {
mi_assert_internal(page->block_size==0||page->block_size >= size);
mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size);
mi_block_t* block = page->free;
if (mi_unlikely(block == NULL)) {
return _mi_malloc_generic(heap, size); // slow path
@ -94,16 +94,16 @@ void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) {
// or the recalloc/rezalloc functions cannot safely expand in place (see issue #63)
UNUSED(size);
mi_assert_internal(p != NULL);
mi_assert_internal(size > 0 && page->block_size >= size);
mi_assert_internal(size > 0 && mi_page_block_size(page) >= size);
mi_assert_internal(_mi_ptr_page(p)==page);
if (page->is_zero) {
// already zero initialized memory?
((mi_block_t*)p)->next = 0; // clear the free list pointer
mi_assert_expensive(mi_mem_is_zero(p,page->block_size));
mi_assert_expensive(mi_mem_is_zero(p, mi_page_block_size(page)));
}
else {
// otherwise memset
memset(p, 0, page->block_size);
memset(p, 0, mi_page_block_size(page));
}
}
@ -141,13 +141,12 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons
static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {
// The decoded value is in the same page (or NULL).
// Walk the free lists to verify positively if it is already freed
mi_thread_free_t tf = (mi_thread_free_t)mi_atomic_read_relaxed(mi_atomic_cast(uintptr_t, &page->thread_free));
// Walk the free lists to verify positively if it is already freed
if (mi_list_contains(page, page->free, block) ||
mi_list_contains(page, page->local_free, block) ||
mi_list_contains(page, mi_tf_block(tf), block))
mi_list_contains(page, mi_page_thread_free(page), block))
{
_mi_fatal_error("double free detected of block %p with size %zu\n", block, page->block_size);
_mi_fatal_error("double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
return true;
}
return false;
@ -177,44 +176,50 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block
// Free
// ------------------------------------------------------
// free huge block from another thread
static mi_decl_noinline void mi_free_huge_block_mt(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) {
// huge page segments are always abandoned and can be freed immediately
mi_assert_internal(segment->page_kind==MI_PAGE_HUGE);
mi_assert_internal(segment == _mi_page_segment(page));
mi_assert_internal(mi_atomic_read_relaxed(&segment->thread_id)==0);
// claim it and free
mi_heap_t* heap = mi_get_default_heap();
// paranoia: if this it the last reference, the cas should always succeed
if (mi_atomic_cas_strong(&segment->thread_id, heap->thread_id, 0)) {
mi_block_set_next(page, block, page->free);
page->free = block;
page->used--;
page->is_zero = false;
mi_assert(page->used == 0);
mi_tld_t* tld = heap->tld;
const size_t bsize = mi_page_block_size(page);
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&tld->stats.giant, bsize);
}
else {
_mi_stat_decrease(&tld->stats.huge, bsize);
}
_mi_segment_page_free(page, true, &tld->segments);
}
}
// multi-threaded free
static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block)
{
mi_thread_free_t tfree;
mi_thread_free_t tfreex;
bool use_delayed;
// huge page segments are always abandoned and can be freed immediately
mi_segment_t* segment = _mi_page_segment(page);
if (segment->page_kind==MI_PAGE_HUGE) {
// huge page segments are always abandoned and can be freed immediately
mi_assert_internal(mi_atomic_read_relaxed(&segment->thread_id)==0);
mi_assert_internal(mi_atomic_read_ptr_relaxed(mi_atomic_cast(void*,&segment->abandoned_next))==NULL);
// claim it and free
mi_heap_t* heap = mi_get_default_heap();
// paranoia: if this it the last reference, the cas should always succeed
if (mi_atomic_cas_strong(&segment->thread_id,heap->thread_id,0)) {
mi_block_set_next(page, block, page->free);
page->free = block;
page->used--;
page->is_zero = false;
mi_assert(page->used == 0);
mi_tld_t* tld = heap->tld;
if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&tld->stats.giant, page->block_size);
}
else {
_mi_stat_decrease(&tld->stats.huge, page->block_size);
}
_mi_segment_page_free(page,true,&tld->segments);
}
mi_free_huge_block_mt(segment, page, block);
return;
}
mi_thread_free_t tfree;
mi_thread_free_t tfreex;
bool use_delayed;
do {
tfree = page->thread_free;
use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE ||
(mi_tf_delayed(tfree) == MI_NO_DELAYED_FREE && page->used == mi_atomic_read_relaxed(&page->thread_freed)+1) // data-race but ok, just optimizes early release of the page
);
tfree = mi_atomic_read_relaxed(&page->xthread_free);
use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE);
if (mi_unlikely(use_delayed)) {
// unlikely: this only happens on the first concurrent free in a page that is in the full list
tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING);
@ -224,15 +229,11 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
mi_block_set_next(page, block, mi_tf_block(tfree));
tfreex = mi_tf_set_block(tfree,block);
}
} while (!mi_atomic_cas_weak(mi_atomic_cast(uintptr_t,&page->thread_free), tfreex, tfree));
} while (!mi_atomic_cas_weak(&page->xthread_free, tfreex, tfree));
if (mi_likely(!use_delayed)) {
// increment the thread free count and return
mi_atomic_increment(&page->thread_freed);
}
else {
if (mi_unlikely(use_delayed)) {
// racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`)
mi_heap_t* heap = (mi_heap_t*)mi_atomic_read_ptr(mi_atomic_cast(void*, &page->heap));
mi_heap_t* heap = mi_page_heap(page);
mi_assert_internal(heap != NULL);
if (heap != NULL) {
// add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity)
@ -245,10 +246,10 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
// and reset the MI_DELAYED_FREEING flag
do {
tfreex = tfree = page->thread_free;
mi_assert_internal(mi_tf_delayed(tfree) == MI_NEVER_DELAYED_FREE || mi_tf_delayed(tfree) == MI_DELAYED_FREEING);
if (mi_tf_delayed(tfree) != MI_NEVER_DELAYED_FREE) tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE);
} while (!mi_atomic_cas_weak(mi_atomic_cast(uintptr_t,&page->thread_free), tfreex, tfree));
tfreex = tfree = mi_atomic_read_relaxed(&page->xthread_free);
mi_assert_internal(mi_tf_delayed(tfree) == MI_DELAYED_FREEING);
tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE);
} while (!mi_atomic_cas_weak(&page->xthread_free, tfreex, tfree));
}
}
@ -257,7 +258,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block)
{
#if (MI_DEBUG)
memset(block, MI_DEBUG_FREED, page->block_size);
memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
#endif
// and push it on the free list
@ -284,7 +285,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) {
mi_assert_internal(page!=NULL && p!=NULL);
size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL);
size_t adjust = (diff % page->block_size);
size_t adjust = (diff % mi_page_block_size(page));
return (mi_block_t*)((uintptr_t)p - adjust);
}
@ -329,8 +330,8 @@ void mi_free(void* p) mi_attr_noexcept
#if (MI_STAT>1)
mi_heap_t* heap = mi_heap_get_default();
mi_heap_stat_decrease(heap, malloc, mi_usable_size(p));
if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap, normal[_mi_bin(page->block_size)], 1);
if (page->xblock_size <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap, normal[_mi_bin(page->xblock_size)], 1);
}
// huge page stat is accounted for in `_mi_page_retire`
#endif
@ -342,7 +343,9 @@ void mi_free(void* p) mi_attr_noexcept
mi_block_set_next(page, block, page->local_free);
page->local_free = block;
page->used--;
if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); }
if (mi_unlikely(mi_page_all_free(page))) {
_mi_page_retire(page);
}
}
else {
// non-local, aligned blocks, or a full page; use the more generic path
@ -356,13 +359,19 @@ bool _mi_free_delayed_block(mi_block_t* block) {
mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(_mi_thread_id() == segment->thread_id);
mi_page_t* page = _mi_segment_page_of(segment, block);
if (mi_tf_delayed(page->thread_free) == MI_DELAYED_FREEING) {
// we might already start delayed freeing while another thread has not yet
// reset the delayed_freeing flag; in that case don't free it quite yet if
// this is the last block remaining.
if (page->used - page->thread_freed == 1) return false;
}
_mi_free_block(page,true,block);
// Clear the no-delayed flag so delayed freeing is used again for this page.
// This must be done before collecting the free lists on this page -- otherwise
// some blocks may end up in the page `thread_free` list with no blocks in the
// heap `thread_delayed_free` list which may cause the page to be never freed!
// (it would only be freed if we happen to scan it in `mi_page_queue_find_free_ex`)
_mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */);
// collect all other non-local frees to ensure up-to-date `used` count
_mi_page_free_collect(page, false);
// and free the block (possibly freeing the page as well since used is updated)
_mi_free_block(page, true, block);
return true;
}
@ -371,7 +380,7 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept {
if (p==NULL) return 0;
const mi_segment_t* segment = _mi_ptr_segment(p);
const mi_page_t* page = _mi_segment_page_of(segment,p);
size_t size = page->block_size;
size_t size = mi_page_block_size(page);
if (mi_unlikely(mi_page_has_aligned(page))) {
ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)_mi_page_ptr_unalign(segment,page,p);
mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);

67
src/heap.c
View File

@ -34,7 +34,7 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void
mi_page_t* page = pq->first;
while(page != NULL) {
mi_page_t* next = page->next; // save next in case the page gets removed from the queue
mi_assert_internal(page->heap == heap);
mi_assert_internal(mi_page_heap(page) == heap);
count++;
if (!fn(heap, pq, page, arg1, arg2)) return false;
page = next; // and continue
@ -50,7 +50,7 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
UNUSED(arg1);
UNUSED(arg2);
UNUSED(pq);
mi_assert_internal(page->heap == heap);
mi_assert_internal(mi_page_heap(page) == heap);
mi_segment_t* segment = _mi_page_segment(page);
mi_assert_internal(segment->thread_id == heap->thread_id);
mi_assert_expensive(_mi_page_is_valid(page));
@ -118,13 +118,18 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
// this may free some segments (but also take ownership of abandoned pages)
_mi_segment_try_reclaim_abandoned(heap, false, &heap->tld->segments);
}
#if MI_DEBUG
else if (collect == ABANDON && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
else if (
#ifdef NDEBUG
collect == FORCE
#else
collect >= FORCE
#endif
&& _mi_is_main_thread() && mi_heap_is_backing(heap))
{
// the main thread is abandoned, try to free all abandoned segments.
// if all memory is freed by now, all segments should be freed.
_mi_segment_try_reclaim_abandoned(heap, true, &heap->tld->segments);
}
#endif
}
// if abandoning, mark all pages to no longer add to delayed_free
@ -245,25 +250,27 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);
// stats
if (page->block_size > MI_LARGE_OBJ_SIZE_MAX) {
if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&heap->tld->stats.giant,page->block_size);
const size_t bsize = mi_page_block_size(page);
if (bsize > MI_LARGE_OBJ_SIZE_MAX) {
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&heap->tld->stats.giant, bsize);
}
else {
_mi_stat_decrease(&heap->tld->stats.huge, page->block_size);
_mi_stat_decrease(&heap->tld->stats.huge, bsize);
}
}
#if (MI_STAT>1)
size_t inuse = page->used - page->thread_freed;
if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse);
#if (MI_STAT>1)
_mi_page_free_collect(page, false); // update used count
const size_t inuse = page->used;
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap, normal[_mi_bin(bsize)], inuse);
}
mi_heap_stat_decrease(heap,malloc, page->block_size * inuse); // todo: off for aligned blocks...
#endif
mi_heap_stat_decrease(heap, malloc, bsize * inuse); // todo: off for aligned blocks...
#endif
// pretend it is all free now
mi_assert_internal(page->thread_freed<=0xFFFF);
page->used = (uint16_t)page->thread_freed;
/// pretend it is all free now
mi_assert_internal(mi_page_thread_free(page) == NULL);
page->used = 0;
// and free the page
_mi_segment_page_free(page,false /* no force? */, &heap->tld->segments);
@ -374,7 +381,7 @@ static mi_heap_t* mi_heap_of_block(const void* p) {
bool valid = (_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(valid);
if (mi_unlikely(!valid)) return NULL;
return _mi_segment_page_of(segment,p)->heap;
return mi_page_heap(_mi_segment_page_of(segment,p));
}
bool mi_heap_contains_block(mi_heap_t* heap, const void* p) {
@ -390,7 +397,7 @@ static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
bool* found = (bool*)vfound;
mi_segment_t* segment = _mi_page_segment(page);
void* start = _mi_page_start(segment, page, NULL);
void* end = (uint8_t*)start + (page->capacity * page->block_size);
void* end = (uint8_t*)start + (page->capacity * mi_page_block_size(page));
*found = (p >= start && p < end);
return (!*found); // continue if not found
}
@ -432,13 +439,14 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
mi_assert_internal(page->local_free == NULL);
if (page->used == 0) return true;
const size_t bsize = mi_page_block_size(page);
size_t psize;
uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize);
if (page->capacity == 1) {
// optimize page with one block
mi_assert_internal(page->used == 1 && page->free == NULL);
return visitor(page->heap, area, pstart, page->block_size, arg);
return visitor(mi_page_heap(page), area, pstart, bsize, arg);
}
// create a bitmap of free blocks.
@ -451,8 +459,8 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
free_count++;
mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize));
size_t offset = (uint8_t*)block - pstart;
mi_assert_internal(offset % page->block_size == 0);
size_t blockidx = offset / page->block_size; // Todo: avoid division?
mi_assert_internal(offset % bsize == 0);
size_t blockidx = offset / bsize; // Todo: avoid division?
mi_assert_internal( blockidx < MI_MAX_BLOCKS);
size_t bitidx = (blockidx / sizeof(uintptr_t));
size_t bit = blockidx - (bitidx * sizeof(uintptr_t));
@ -471,8 +479,8 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
}
else if ((m & ((uintptr_t)1 << bit)) == 0) {
used_count++;
uint8_t* block = pstart + (i * page->block_size);
if (!visitor(page->heap, area, block, page->block_size, arg)) return false;
uint8_t* block = pstart + (i * bsize);
if (!visitor(mi_page_heap(page), area, block, bsize, arg)) return false;
}
}
mi_assert_internal(page->used == used_count);
@ -487,12 +495,13 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
UNUSED(pq);
mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun;
mi_heap_area_ex_t xarea;
const size_t bsize = mi_page_block_size(page);
xarea.page = page;
xarea.area.reserved = page->reserved * page->block_size;
xarea.area.committed = page->capacity * page->block_size;
xarea.area.reserved = page->reserved * bsize;
xarea.area.committed = page->capacity * bsize;
xarea.area.blocks = _mi_page_start(_mi_page_segment(page), page, NULL);
xarea.area.used = page->used - page->thread_freed; // race is ok
xarea.area.block_size = page->block_size;
xarea.area.used = page->used;
xarea.area.block_size = bsize;
return fun(heap, &xarea, arg);
}

13
src/init.c
View File

@ -23,12 +23,11 @@ const mi_page_t _mi_page_empty = {
{ 0, 0 },
#endif
0, // used
NULL,
ATOMIC_VAR_INIT(0), ATOMIC_VAR_INIT(0),
0, NULL, NULL, NULL
#if (MI_INTPTR_SIZE==4)
, { NULL } // padding
#endif
0, // xblock_size
NULL, // local_free
ATOMIC_VAR_INIT(0), // xthread_free
ATOMIC_VAR_INIT(0), // xheap
NULL, NULL
};
#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
@ -393,7 +392,7 @@ static void mi_process_load(void) {
const char* msg = NULL;
mi_allocator_init(&msg);
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
_mi_fputs(NULL,NULL,msg);
_mi_fputs(NULL,NULL,NULL,msg);
}
}

61
src/options.c
View File

@ -67,7 +67,6 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
{ 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread
{ 1, UNINIT, MI_OPTION(page_reset) }, // reset pages on free
{ 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit)
{ 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
{ 100, UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds
@ -140,7 +139,8 @@ void mi_option_disable(mi_option_t option) {
}
static void mi_out_stderr(const char* msg) {
static void mi_out_stderr(const char* msg, void* arg) {
UNUSED(arg);
#ifdef _WIN32
// on windows with redirection, the C runtime cannot handle locale dependent output
// after the main thread closes so we use direct console output.
@ -160,7 +160,8 @@ static void mi_out_stderr(const char* msg) {
static char out_buf[MI_MAX_DELAY_OUTPUT+1];
static _Atomic(uintptr_t) out_len;
static void mi_out_buf(const char* msg) {
static void mi_out_buf(const char* msg, void* arg) {
UNUSED(arg);
if (msg==NULL) return;
if (mi_atomic_read_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return;
size_t n = strlen(msg);
@ -175,14 +176,14 @@ static void mi_out_buf(const char* msg) {
memcpy(&out_buf[start], msg, n);
}
static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf) {
static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
if (out==NULL) return;
// claim (if `no_more_buf == true`, no more output will be added after this point)
size_t count = mi_atomic_addu(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));
// and output the current contents
if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
out_buf[count] = 0;
out(out_buf);
out(out_buf,arg);
if (!no_more_buf) {
out_buf[count] = '\n'; // if continue with the buffer, insert a newline
}
@ -191,9 +192,9 @@ static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf) {
// Once this module is loaded, switch to this routine
// which outputs to stderr and the delayed output buffer.
static void mi_out_buf_stderr(const char* msg) {
mi_out_stderr(msg);
mi_out_buf(msg);
static void mi_out_buf_stderr(const char* msg, void* arg) {
mi_out_stderr(msg,arg);
mi_out_buf(msg,arg);
}
@ -206,21 +207,25 @@ static void mi_out_buf_stderr(const char* msg) {
// For now, don't register output from multiple threads.
#pragma warning(suppress:4180)
static mi_output_fun* volatile mi_out_default; // = NULL
static volatile _Atomic(void*) mi_out_arg; // = NULL
static mi_output_fun* mi_out_get_default(void) {
static mi_output_fun* mi_out_get_default(void** parg) {
if (parg != NULL) { *parg = mi_atomic_read_ptr(&mi_out_arg); }
mi_output_fun* out = mi_out_default;
return (out == NULL ? &mi_out_buf : out);
}
void mi_register_output(mi_output_fun* out) mi_attr_noexcept {
void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept {
mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer
if (out!=NULL) mi_out_buf_flush(out,true); // output all the delayed output now
mi_atomic_write_ptr(&mi_out_arg, arg);
if (out!=NULL) mi_out_buf_flush(out,true,arg); // output all the delayed output now
}
// add stderr to the delayed output after the module is loaded
static void mi_add_stderr_output() {
mi_out_buf_flush(&mi_out_stderr, false); // flush current contents to stderr
mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output
mi_assert_internal(mi_out_default == NULL);
mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr
mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output
}
// --------------------------------------------------------
@ -232,33 +237,35 @@ static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT
// inside the C runtime causes another message.
static mi_decl_thread bool recurse = false;
void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message) {
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
if (recurse) return;
if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) out = mi_out_get_default();
if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) { // TODO: use mi_out_stderr for stderr?
out = mi_out_get_default(&arg);
}
recurse = true;
if (prefix != NULL) out(prefix);
out(message);
if (prefix != NULL) out(prefix,arg);
out(message,arg);
recurse = false;
return;
}
// Define our own limited `fprintf` that avoids memory allocation.
// We do this using `snprintf` with a limited buffer.
static void mi_vfprintf( mi_output_fun* out, const char* prefix, const char* fmt, va_list args ) {
static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {
char buf[512];
if (fmt==NULL) return;
if (recurse) return;
recurse = true;
vsnprintf(buf,sizeof(buf)-1,fmt,args);
recurse = false;
_mi_fputs(out,prefix,buf);
_mi_fputs(out,arg,prefix,buf);
}
void _mi_fprintf( mi_output_fun* out, const char* fmt, ... ) {
void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
va_list args;
va_start(args,fmt);
mi_vfprintf(out,NULL,fmt,args);
mi_vfprintf(out,arg,NULL,fmt,args);
va_end(args);
}
@ -266,7 +273,7 @@ void _mi_trace_message(const char* fmt, ...) {
if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher
va_list args;
va_start(args, fmt);
mi_vfprintf(NULL, "mimalloc: ", fmt, args);
mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args);
va_end(args);
}
@ -274,7 +281,7 @@ void _mi_verbose_message(const char* fmt, ...) {
if (!mi_option_is_enabled(mi_option_verbose)) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(NULL, "mimalloc: ", fmt, args);
mi_vfprintf(NULL, NULL, "mimalloc: ", fmt, args);
va_end(args);
}
@ -283,7 +290,7 @@ void _mi_error_message(const char* fmt, ...) {
if (mi_atomic_increment(&error_count) > mi_max_error_count) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(NULL, "mimalloc: error: ", fmt, args);
mi_vfprintf(NULL, NULL, "mimalloc: error: ", fmt, args);
va_end(args);
mi_assert(false);
}
@ -293,14 +300,14 @@ void _mi_warning_message(const char* fmt, ...) {
if (mi_atomic_increment(&error_count) > mi_max_error_count) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(NULL, "mimalloc: warning: ", fmt, args);
mi_vfprintf(NULL, NULL, "mimalloc: warning: ", fmt, args);
va_end(args);
}
#if MI_DEBUG
void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) {
_mi_fprintf(NULL,"mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
_mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
abort();
}
#endif
@ -308,7 +315,7 @@ void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, co
mi_attr_noreturn void _mi_fatal_error(const char* fmt, ...) {
va_list args;
va_start(args, fmt);
mi_vfprintf(NULL, "mimalloc: fatal: ", fmt, args);
mi_vfprintf(NULL, NULL, "mimalloc: fatal: ", fmt, args);
va_end(args);
#if (MI_SECURE>=0)
abort();

44
src/page-queue.c
View File

@ -178,20 +178,20 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t*
#endif
static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {
uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->block_size));
mi_heap_t* heap = page->heap;
uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size));
mi_heap_t* heap = mi_page_heap(page);
mi_assert_internal(heap != NULL && bin <= MI_BIN_FULL);
mi_page_queue_t* pq = &heap->pages[bin];
mi_assert_internal(bin >= MI_BIN_HUGE || page->block_size == pq->block_size);
mi_assert_internal(bin >= MI_BIN_HUGE || page->xblock_size == pq->block_size);
mi_assert_expensive(mi_page_queue_contains(pq, page));
return pq;
}
static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) {
uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->block_size));
uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size));
mi_assert_internal(bin <= MI_BIN_FULL);
mi_page_queue_t* pq = &heap->pages[bin];
mi_assert_internal(mi_page_is_in_full(page) || page->block_size == pq->block_size);
mi_assert_internal(mi_page_is_in_full(page) || page->xblock_size == pq->block_size);
return pq;
}
@ -246,35 +246,35 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) {
static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_expensive(mi_page_queue_contains(queue, page));
mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
mi_assert_internal(page->xblock_size == queue->block_size || (page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
mi_heap_t* heap = mi_page_heap(page);
if (page->prev != NULL) page->prev->next = page->next;
if (page->next != NULL) page->next->prev = page->prev;
if (page == queue->last) queue->last = page->prev;
if (page == queue->first) {
queue->first = page->next;
// update first
mi_heap_t* heap = page->heap;
mi_assert_internal(mi_heap_contains_queue(heap, queue));
mi_heap_queue_first_update(heap,queue);
}
page->heap->page_count--;
heap->page_count--;
page->next = NULL;
page->prev = NULL;
mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL);
// mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL);
mi_page_set_in_full(page,false);
}
static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) {
mi_assert_internal(page->heap == NULL);
mi_assert_internal(mi_page_heap(page) == heap);
mi_assert_internal(!mi_page_queue_contains(queue, page));
mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
mi_assert_internal(page->block_size == queue->block_size ||
(page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) ||
mi_assert_internal(page->xblock_size == queue->block_size ||
(page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) ||
(mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
mi_page_set_in_full(page, mi_page_queue_is_full(queue));
mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), heap);
// mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), heap);
page->next = queue->first;
page->prev = NULL;
if (queue->first != NULL) {
@ -296,19 +296,19 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro
mi_assert_internal(page != NULL);
mi_assert_expensive(mi_page_queue_contains(from, page));
mi_assert_expensive(!mi_page_queue_contains(to, page));
mi_assert_internal((page->block_size == to->block_size && page->block_size == from->block_size) ||
(page->block_size == to->block_size && mi_page_queue_is_full(from)) ||
(page->block_size == from->block_size && mi_page_queue_is_full(to)) ||
(page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) ||
(page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_full(to)));
mi_assert_internal((page->xblock_size == to->block_size && page->xblock_size == from->block_size) ||
(page->xblock_size == to->block_size && mi_page_queue_is_full(from)) ||
(page->xblock_size == from->block_size && mi_page_queue_is_full(to)) ||
(page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) ||
(page->xblock_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_full(to)));
mi_heap_t* heap = mi_page_heap(page);
if (page->prev != NULL) page->prev->next = page->next;
if (page->next != NULL) page->next->prev = page->prev;
if (page == from->last) from->last = page->prev;
if (page == from->first) {
from->first = page->next;
// update first
mi_heap_t* heap = page->heap;
mi_assert_internal(mi_heap_contains_queue(heap, from));
mi_heap_queue_first_update(heap, from);
}
@ -316,14 +316,14 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro
page->prev = to->last;
page->next = NULL;
if (to->last != NULL) {
mi_assert_internal(page->heap == to->last->heap);
mi_assert_internal(heap == mi_page_heap(to->last));
to->last->next = page;
to->last = page;
}
else {
to->first = page;
to->last = page;
mi_heap_queue_first_update(page->heap, to);
mi_heap_queue_first_update(heap, to);
}
mi_page_set_in_full(page, mi_page_queue_is_full(to));
@ -338,7 +338,7 @@ size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue
// set append pages to new heap and count
size_t count = 0;
for (mi_page_t* page = append->first; page != NULL; page = page->next) {
mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), heap);
mi_page_set_heap(page,heap);
count++;
}

194
src/page.c
View File

@ -29,10 +29,11 @@ terms of the MIT license. A copy of the license can be found in the file
----------------------------------------------------------- */
// Index a block in a page
static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t i) {
static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) {
UNUSED(page);
mi_assert_internal(page != NULL);
mi_assert_internal(i <= page->reserved);
return (mi_block_t*)((uint8_t*)page_start + (i * page->block_size));
return (mi_block_t*)((uint8_t*)page_start + (i * block_size));
}
static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t* tld);
@ -69,13 +70,14 @@ static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {
}
static bool mi_page_is_valid_init(mi_page_t* page) {
mi_assert_internal(page->block_size > 0);
mi_assert_internal(page->xblock_size > 0);
mi_assert_internal(page->used <= page->capacity);
mi_assert_internal(page->capacity <= page->reserved);
const size_t bsize = mi_page_block_size(page);
mi_segment_t* segment = _mi_page_segment(page);
uint8_t* start = _mi_page_start(segment,page,NULL);
mi_assert_internal(start == _mi_segment_page_start(segment,page,page->block_size,NULL,NULL));
mi_assert_internal(start == _mi_segment_page_start(segment,page,bsize,NULL,NULL));
//mi_assert_internal(start + page->capacity*page->block_size == page->top);
mi_assert_internal(mi_page_list_is_valid(page,page->free));
@ -89,10 +91,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
}
#endif
mi_block_t* tfree = mi_tf_block(page->thread_free);
mi_block_t* tfree = mi_page_thread_free(page);
mi_assert_internal(mi_page_list_is_valid(page, tfree));
size_t tfree_count = mi_page_list_count(page, tfree);
mi_assert_internal(tfree_count <= page->thread_freed + 1);
//size_t tfree_count = mi_page_list_count(page, tfree);
//mi_assert_internal(tfree_count <= page->thread_freed + 1);
size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);
mi_assert_internal(page->used + free_count == page->capacity);
@ -105,14 +107,14 @@ bool _mi_page_is_valid(mi_page_t* page) {
#if MI_SECURE
mi_assert_internal(page->key != 0);
#endif
if (page->heap!=NULL) {
if (mi_page_heap(page)!=NULL) {
mi_segment_t* segment = _mi_page_segment(page);
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id || segment->thread_id==0);
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == mi_page_heap(page)->thread_id || segment->thread_id==0);
if (segment->page_kind != MI_PAGE_HUGE) {
mi_page_queue_t* pq = mi_page_queue_of(page);
mi_assert_internal(mi_page_queue_contains(pq, page));
mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page));
mi_assert_internal(mi_heap_contains_queue(page->heap,pq));
mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page));
mi_assert_internal(mi_heap_contains_queue(mi_page_heap(page),pq));
}
}
return true;
@ -124,20 +126,20 @@ void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool overrid
mi_thread_free_t tfreex;
mi_delayed_t old_delay;
do {
tfree = mi_atomic_read_relaxed(&page->thread_free);
tfree = mi_atomic_read(&page->xthread_free);
tfreex = mi_tf_set_delayed(tfree, delay);
old_delay = mi_tf_delayed(tfree);
if (mi_unlikely(old_delay == MI_DELAYED_FREEING)) {
mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
// mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail
}
else if (delay == old_delay) {
break; // avoid atomic operation if already equal
}
else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE) {
break; // leave never set
break; // leave never-delayed flag set
}
} while ((old_delay == MI_DELAYED_FREEING) ||
!mi_atomic_cas_weak(mi_atomic_cast(uintptr_t, &page->thread_free), tfreex, tfree));
} while (!mi_atomic_cas_weak(&page->xthread_free, tfreex, tfree));
}
/* -----------------------------------------------------------
@ -154,17 +156,17 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
mi_thread_free_t tfree;
mi_thread_free_t tfreex;
do {
tfree = page->thread_free;
tfree = mi_atomic_read_relaxed(&page->xthread_free);
head = mi_tf_block(tfree);
tfreex = mi_tf_set_block(tfree,NULL);
} while (!mi_atomic_cas_weak(mi_atomic_cast(uintptr_t,&page->thread_free), tfreex, tfree));
} while (!mi_atomic_cas_weak(&page->xthread_free, tfreex, tfree));
// return if the list is empty
if (head == NULL) return;
// find the tail -- also to get a proper count (without data races)
uintptr_t max_count = page->capacity; // cannot collect more than capacity
uintptr_t count = 1;
uint32_t max_count = page->capacity; // cannot collect more than capacity
uint32_t count = 1;
mi_block_t* tail = head;
mi_block_t* next;
while ((next = mi_block_next(page,tail)) != NULL && count <= max_count) {
@ -182,7 +184,6 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
page->local_free = head;
// update counts now
mi_atomic_subu(&page->thread_freed, count);
page->used -= count;
}
@ -190,7 +191,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
mi_assert_internal(page!=NULL);
// collect the thread free list
if (force || mi_tf_block(page->thread_free) != NULL) { // quick test to avoid an atomic operation
if (force || mi_page_thread_free(page) != NULL) { // quick test to avoid an atomic operation
_mi_page_thread_free_collect(page);
}
@ -228,15 +229,12 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
// called from segments when reclaiming abandoned pages
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {
mi_assert_expensive(mi_page_is_valid_init(page));
mi_assert_internal(page->heap == NULL);
mi_assert_internal(mi_page_heap(page) == heap);
mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);
mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
mi_assert_internal(!page->is_reset);
mi_assert_internal(mi_tf_delayed(page->thread_free) == MI_NEVER_DELAYED_FREE);
_mi_page_free_collect(page,false);
mi_page_queue_t* pq = mi_page_queue(heap, page->block_size);
mi_page_queue_push(heap, pq, page);
mi_assert_internal(page->heap != NULL);
_mi_page_use_delayed_free(page, MI_NO_DELAYED_FREE, true); // override never (after push so heap is set)
mi_page_queue_t* pq = mi_page_queue(heap, mi_page_block_size(page));
mi_page_queue_push(heap, pq, page);
mi_assert_expensive(_mi_page_is_valid(page));
}
@ -270,8 +268,8 @@ static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) {
// otherwise allocate the page
page = mi_page_fresh_alloc(heap, pq, pq->block_size);
if (page==NULL) return NULL;
mi_assert_internal(pq->block_size==page->block_size);
mi_assert_internal(pq==mi_page_queue(heap,page->block_size));
mi_assert_internal(pq->block_size==mi_page_block_size(page));
mi_assert_internal(pq==mi_page_queue(heap, mi_page_block_size(page)));
return page;
}
@ -312,11 +310,9 @@ void _mi_page_unfull(mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(mi_page_is_in_full(page));
_mi_page_use_delayed_free(page, MI_NO_DELAYED_FREE, false);
if (!mi_page_is_in_full(page)) return;
mi_heap_t* heap = page->heap;
mi_heap_t* heap = mi_page_heap(page);
mi_page_queue_t* pqfull = &heap->pages[MI_BIN_FULL];
mi_page_set_in_full(page, false); // to get the right queue
mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);
@ -329,10 +325,8 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
mi_assert_internal(!mi_page_immediate_available(page));
mi_assert_internal(!mi_page_is_in_full(page));
_mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false);
if (mi_page_is_in_full(page)) return;
mi_page_queue_enqueue_from(&page->heap->pages[MI_BIN_FULL], pq, page);
mi_page_queue_enqueue_from(&mi_page_heap(page)->pages[MI_BIN_FULL], pq, page);
_mi_page_free_collect(page,false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
}
@ -345,18 +339,17 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(page->heap != NULL);
mi_assert_internal(mi_page_heap(page) != NULL);
#if MI_DEBUG > 1
mi_heap_t* pheap = (mi_heap_t*)mi_atomic_read_ptr(mi_atomic_cast(void*, &page->heap));
#endif
mi_heap_t* pheap = mi_page_heap(page);
// remove from our page list
mi_segments_tld_t* segments_tld = &page->heap->tld->segments;
mi_segments_tld_t* segments_tld = &pheap->tld->segments;
mi_page_queue_remove(pq, page);
// page is no longer associated with our heap
mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL);
mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE);
mi_page_set_heap(page, NULL);
#if MI_DEBUG>1
// check there are no references left..
@ -366,7 +359,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
#endif
// and abandon it
mi_assert_internal(page->heap == NULL);
mi_assert_internal(mi_page_heap(page) == NULL);
_mi_segment_page_abandon(page,segments_tld);
}
@ -377,33 +370,18 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(mi_page_all_free(page));
#if MI_DEBUG>1
// check if we can safely free
mi_thread_free_t free = mi_tf_set_delayed(page->thread_free,MI_NEVER_DELAYED_FREE);
free = mi_atomic_exchange(&page->thread_free, free);
mi_assert_internal(mi_tf_delayed(free) != MI_DELAYED_FREEING);
#endif
mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING);
// no more aligned blocks in here
mi_page_set_has_aligned(page, false);
// account for huge pages here
// (note: no longer necessary as huge pages are always abandoned)
if (page->block_size > MI_LARGE_OBJ_SIZE_MAX) {
if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&page->heap->tld->stats.giant, page->block_size);
}
else {
_mi_stat_decrease(&page->heap->tld->stats.huge, page->block_size);
}
}
// remove from the page list
// (no need to do _mi_heap_delayed_free first as all blocks are already free)
mi_segments_tld_t* segments_tld = &page->heap->tld->segments;
mi_segments_tld_t* segments_tld = &mi_page_heap(page)->tld->segments;
mi_page_queue_remove(pq, page);
// and free it
mi_assert_internal(page->heap == NULL);
mi_page_set_heap(page,NULL);
_mi_segment_page_free(page, force, segments_tld);
}
@ -427,7 +405,7 @@ void _mi_page_retire(mi_page_t* page) {
// how to check this efficiently though...
// for now, we don't retire if it is the only page left of this size class.
mi_page_queue_t* pq = mi_page_queue_of(page);
if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) {
if (mi_likely(page->xblock_size <= MI_SMALL_SIZE_MAX && !mi_page_is_in_full(page))) {
if (pq->last==page && pq->first==page) { // the only page in the queue?
mi_stat_counter_increase(_mi_stats_main.page_no_retire,1);
page->retire_expire = 4;
@ -469,15 +447,15 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) {
#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT)
#define MI_MIN_SLICES (2)
static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t extend, mi_stats_t* const stats) {
static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) {
UNUSED(stats);
#if (MI_SECURE<=2)
mi_assert_internal(page->free == NULL);
mi_assert_internal(page->local_free == NULL);
#endif
mi_assert_internal(page->capacity + extend <= page->reserved);
mi_assert_internal(bsize == mi_page_block_size(page));
void* const page_area = _mi_page_start(_mi_page_segment(page), page, NULL);
const size_t bsize = page->block_size;
// initialize a randomized free list
// set up `slice_count` slices to alternate between
@ -491,7 +469,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice
size_t counts[MI_MAX_SLICES]; // available objects in the slice
for (size_t i = 0; i < slice_count; i++) {
blocks[i] = mi_page_block_at(page, page_area, page->capacity + i*slice_extend);
blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend);
counts[i] = slice_extend;
}
counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?)
@ -526,7 +504,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
page->free = free_start;
}
static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t extend, mi_stats_t* const stats)
static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats)
{
UNUSED(stats);
#if (MI_SECURE <= 2)
@ -534,12 +512,13 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
mi_assert_internal(page->local_free == NULL);
#endif
mi_assert_internal(page->capacity + extend <= page->reserved);
mi_assert_internal(bsize == mi_page_block_size(page));
void* const page_area = _mi_page_start(_mi_page_segment(page), page, NULL );
const size_t bsize = page->block_size;
mi_block_t* const start = mi_page_block_at(page, page_area, page->capacity);
mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity);
// initialize a sequential free list
mi_block_t* const last = mi_page_block_at(page, page_area, page->capacity + extend - 1);
mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);
mi_block_t* block = start;
while(block <= last) {
mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize);
@ -581,8 +560,9 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
mi_stat_counter_increase(tld->stats.pages_extended, 1);
// calculate the extend count
const size_t bsize = (page->xblock_size < MI_HUGE_BLOCK_SIZE ? page->xblock_size : page_size);
size_t extend = page->reserved - page->capacity;
size_t max_extend = (page->block_size >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/(uint32_t)page->block_size);
size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/(uint32_t)bsize);
if (max_extend < MI_MIN_EXTEND) max_extend = MI_MIN_EXTEND;
if (extend > max_extend) {
@ -596,20 +576,20 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
// commit on-demand for large and huge pages?
if (_mi_page_segment(page)->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) {
uint8_t* start = page_start + (page->capacity * page->block_size);
_mi_mem_commit(start, extend * page->block_size, NULL, &tld->os);
uint8_t* start = page_start + (page->capacity * bsize);
_mi_mem_commit(start, extend * bsize, NULL, &tld->os);
}
// and append the extend the free list
if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) {
mi_page_free_list_extend(page, extend, &tld->stats );
mi_page_free_list_extend(page, bsize, extend, &tld->stats );
}
else {
mi_page_free_list_extend_secure(heap, page, extend, &tld->stats);
mi_page_free_list_extend_secure(heap, page, bsize, extend, &tld->stats);
}
// enable the new free list
page->capacity += (uint16_t)extend;
mi_stat_increase(tld->stats.page_committed, extend * page->block_size);
mi_stat_increase(tld->stats.page_committed, extend * bsize);
// extension into zero initialized memory preserves the zero'd free list
if (!page->is_zero_init) {
@ -625,9 +605,10 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert(segment != NULL);
mi_assert_internal(block_size > 0);
// set fields
mi_page_set_heap(page, heap);
size_t page_size;
_mi_segment_page_start(segment, page, block_size, &page_size, NULL);
page->block_size = block_size;
page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE);
mi_assert_internal(page_size / block_size < (1L<<16));
page->reserved = (uint16_t)(page_size / block_size);
#ifdef MI_ENCODE_FREELIST
@ -639,14 +620,14 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert_internal(page->capacity == 0);
mi_assert_internal(page->free == NULL);
mi_assert_internal(page->used == 0);
mi_assert_internal(page->thread_free == 0);
mi_assert_internal(page->thread_freed == 0);
mi_assert_internal(page->xthread_free == 0);
mi_assert_internal(page->next == NULL);
mi_assert_internal(page->prev == NULL);
mi_assert_internal(page->retire_expire == 0);
mi_assert_internal(!mi_page_has_aligned(page));
#if (MI_ENCODE_FREELIST)
mi_assert_internal(page->key != 0);
mi_assert_internal(page->key[1] != 0);
mi_assert_internal(page->key[2] != 0);
#endif
mi_assert_expensive(mi_page_is_valid_init(page));
@ -664,34 +645,19 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* pq)
{
// search through the pages in "next fit" order
mi_page_t* rpage = NULL;
size_t count = 0;
size_t page_free_count = 0;
mi_page_t* page = pq->first;
while( page != NULL)
while (page != NULL)
{
mi_page_t* next = page->next; // remember next
count++;
// 0. collect freed blocks by us and other threads
_mi_page_free_collect(page,false);
_mi_page_free_collect(page, false);
// 1. if the page contains free blocks, we are done
if (mi_page_immediate_available(page)) {
// If all blocks are free, we might retire this page instead.
// do this at most 8 times to bound allocation time.
// (note: this can happen if a page was earlier not retired due
// to having neighbours that were mostly full or due to concurrent frees)
if (page_free_count < 8 && mi_page_all_free(page)) {
page_free_count++;
if (rpage != NULL) _mi_page_free(rpage,pq,false);
rpage = page;
page = next;
continue; // and keep looking
}
else {
break; // pick this one
}
break; // pick this one
}
// 2. Try to extend
@ -704,20 +670,12 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
// 3. If the page is completely full, move it to the `mi_pages_full`
// queue so we don't visit long-lived pages too often.
mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
mi_page_to_full(page,pq);
mi_page_to_full(page, pq);
page = next;
} // for each page
mi_stat_counter_increase(heap->tld->stats.searches,count);
if (page == NULL) {
page = rpage;
rpage = NULL;
}
if (rpage != NULL) {
_mi_page_free(rpage,pq,false);
}
mi_stat_counter_increase(heap->tld->stats.searches, count);
if (page == NULL) {
page = mi_page_fresh(heap, pq);
@ -729,11 +687,12 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
mi_assert_internal(page == NULL || mi_page_immediate_available(page));
// finally collect retired pages
_mi_heap_collect_retired(heap,false);
_mi_heap_collect_retired(heap, false);
return page;
}
// Find a page with free blocks of `size`.
static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
mi_page_queue_t* pq = mi_page_queue(heap,size);
@ -764,18 +723,20 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
----------------------------------------------------------- */
static mi_deferred_free_fun* volatile deferred_free = NULL;
static volatile _Atomic(void*) deferred_arg; // = NULL
void _mi_deferred_free(mi_heap_t* heap, bool force) {
heap->tld->heartbeat++;
if (deferred_free != NULL && !heap->tld->recurse) {
heap->tld->recurse = true;
deferred_free(force, heap->tld->heartbeat);
deferred_free(force, heap->tld->heartbeat, mi_atomic_read_ptr_relaxed(&deferred_arg));
heap->tld->recurse = false;
}
}
void mi_register_deferred_free(mi_deferred_free_fun* fn) mi_attr_noexcept {
void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {
deferred_free = fn;
mi_atomic_write_ptr(&deferred_arg, arg);
}
@ -792,14 +753,15 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE);
mi_page_t* page = mi_page_fresh_alloc(heap,NULL,block_size);
if (page != NULL) {
const size_t bsize = mi_page_block_size(page);
mi_assert_internal(mi_page_immediate_available(page));
mi_assert_internal(page->block_size == block_size);
mi_assert_internal(bsize >= size);
mi_assert_internal(_mi_page_segment(page)->page_kind==MI_PAGE_HUGE);
mi_assert_internal(_mi_page_segment(page)->used==1);
mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue
mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL);
mi_page_set_heap(page, NULL);
if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) {
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_increase(&heap->tld->stats.giant, block_size);
_mi_stat_counter_increase(&heap->tld->stats.giant_count, 1);
}
@ -847,7 +809,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
if (page == NULL) return NULL; // out of memory
mi_assert_internal(mi_page_immediate_available(page));
mi_assert_internal(page->block_size >= size);
mi_assert_internal(mi_page_block_size(page) >= size);
// and try again, this time succeeding! (i.e. this should never recurse)
return _mi_page_malloc(heap, page, size);

47
src/segment.c
View File

@ -236,8 +236,8 @@ static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, m
mi_assert_internal(size <= psize);
size_t reset_size = (size == 0 || size > psize ? psize : size);
if (size == 0 && segment->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) {
mi_assert_internal(page->block_size > 0);
reset_size = page->capacity * page->block_size;
mi_assert_internal(page->xblock_size > 0);
reset_size = page->capacity * mi_page_block_size(page);
}
_mi_mem_reset(start, reset_size, tld->os);
}
@ -251,8 +251,8 @@ static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size,
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
size_t unreset_size = (size == 0 || size > psize ? psize : size);
if (size == 0 && segment->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) {
mi_assert_internal(page->block_size > 0);
unreset_size = page->capacity * page->block_size;
mi_assert_internal(page->xblock_size > 0);
unreset_size = page->capacity * mi_page_block_size(page);
}
bool is_zero = false;
_mi_mem_unreset(start, unreset_size, &is_zero, tld->os);
@ -264,18 +264,18 @@ static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size,
The free page queue
----------------------------------------------------------- */
// we re-use the heap field for the expiration counter. Since this is a
// pointer, it can be 32-bit while the clock is always 64-bit. To guard
// we re-use the `used` field for the expiration counter. Since this is a
// a 32-bit field while the clock is always 64-bit we need to guard
// against overflow, we use substraction to check for expiry which work
// as long as the reset delay is under (2^30 - 1) milliseconds (~12 days)
static void mi_page_reset_set_expire(mi_page_t* page) {
intptr_t expire = (intptr_t)(_mi_clock_now() + mi_option_get(mi_option_reset_delay));
page->heap = (mi_heap_t*)expire;
uint32_t expire = (uint32_t)_mi_clock_now() + mi_option_get(mi_option_reset_delay);
page->used = expire;
}
static bool mi_page_reset_is_expired(mi_page_t* page, mi_msecs_t now) {
intptr_t expire = (intptr_t)(page->heap);
return (((intptr_t)now - expire) >= 0);
int32_t expire = (int32_t)(page->used);
return (((int32_t)now - expire) >= 0);
}
static void mi_pages_reset_add(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) {
@ -320,7 +320,7 @@ static void mi_pages_reset_remove(mi_page_t* page, mi_segments_tld_t* tld) {
if (page == pq->last) pq->last = page->prev;
if (page == pq->first) pq->first = page->next;
page->next = page->prev = NULL;
page->heap = NULL;
page->used = 0;
}
static void mi_pages_reset_remove_all_in_segment(mi_segment_t* segment, mi_segments_tld_t* tld) {
@ -345,7 +345,7 @@ static void mi_reset_delayed(mi_segments_tld_t* tld) {
while (page != NULL && mi_page_reset_is_expired(page,now)) {
mi_page_t* const prev = page->prev; // save previous field
mi_page_reset(_mi_page_segment(page), page, 0, tld);
page->heap = NULL;
page->used = 0;
page->prev = page->next = NULL;
page = prev;
}
@ -386,7 +386,7 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_
}
if (page_size != NULL) *page_size = psize;
mi_assert_internal(page->block_size == 0 || _mi_ptr_page(p) == page);
mi_assert_internal(page->xblock_size == 0 || _mi_ptr_page(p) == page);
mi_assert_internal(_mi_ptr_segment(p) == segment);
return p;
}
@ -409,7 +409,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
}
if (page_size != NULL) *page_size = psize;
mi_assert_internal(page->block_size==0 || _mi_ptr_page(p) == page);
mi_assert_internal(page->xblock_size==0 || _mi_ptr_page(p) == page);
mi_assert_internal(_mi_ptr_segment(p) == segment);
return p;
}
@ -740,7 +740,8 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_seg
mi_assert_internal(mi_page_all_free(page));
mi_assert_internal(page->is_committed);
mi_assert_internal(mi_page_not_in_queue(page, tld));
size_t inuse = page->capacity * page->block_size;
size_t inuse = page->capacity * mi_page_block_size(page);
_mi_stat_decrease(&tld->stats->page_committed, inuse);
_mi_stat_decrease(&tld->stats->pages, 1);
@ -757,10 +758,10 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_seg
//mi_page_reset(segment, page, 0 /*used_size*/, tld);
// zero the page data, but not the segment fields and block_size (for page size calculations)
size_t block_size = page->block_size;
uint32_t block_size = page->xblock_size;
ptrdiff_t ofs = offsetof(mi_page_t,capacity);
memset((uint8_t*)page + ofs, 0, sizeof(*page) - ofs);
page->block_size = block_size;
page->xblock_size = block_size;
segment->used--;
// add to the free page list for reuse/reset
@ -852,6 +853,8 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) {
mi_assert(page != NULL);
mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE);
mi_assert_internal(mi_page_heap(page) == NULL);
mi_segment_t* segment = _mi_page_segment(page);
mi_assert_expensive(!mi_pages_reset_contains(page, tld));
mi_assert_expensive(mi_segment_is_valid(segment,tld));
@ -912,15 +915,21 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
mi_assert_internal(!page->is_reset);
mi_assert_internal(page->is_committed);
mi_assert_internal(mi_page_not_in_queue(page, tld));
mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE);
mi_assert_internal(mi_page_heap(page) == NULL);
segment->abandoned--;
mi_assert(page->next == NULL);
_mi_stat_decrease(&tld->stats->pages_abandoned, 1);
// set the heap again and allow delayed free again
mi_page_set_heap(page, heap);
_mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set)
_mi_page_free_collect(page, false); // ensure used count is up to date
if (mi_page_all_free(page)) {
// if everything free by now, free the page
// if everything free already, clear the page directly
mi_segment_page_clear(segment,page,tld);
}
else {
// otherwise reclaim it
// otherwise reclaim it into the heap
_mi_page_reclaim(heap,page);
}
}

160
src/stats.c
View File

@ -126,7 +126,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
// unit > 0 : size in binary bytes
// unit == 0: count as decimal
// unit < 0 : count in binary
static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, const char* fmt) {
static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {
char buf[32];
int len = 32;
const char* suffix = (unit <= 0 ? " " : "b");
@ -147,75 +147,75 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, const
const long frac1 = (long)(tens%10);
snprintf(buf, len, "%ld.%ld %s%s", whole, frac1, magnitude, suffix);
}
_mi_fprintf(out, (fmt==NULL ? "%11s" : fmt), buf);
_mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf);
}
static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out) {
mi_printf_amount(n,unit,out,NULL);
static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
mi_printf_amount(n,unit,out,arg,NULL);
}
static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out) {
if (unit==1) _mi_fprintf(out,"%11s"," ");
else mi_print_amount(n,0,out);
static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
if (unit==1) _mi_fprintf(out, arg, "%11s"," ");
else mi_print_amount(n,0,out,arg);
}
static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out ) {
_mi_fprintf(out,"%10s:", msg);
static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg ) {
_mi_fprintf(out, arg,"%10s:", msg);
if (unit>0) {
mi_print_amount(stat->peak, unit, out);
mi_print_amount(stat->allocated, unit, out);
mi_print_amount(stat->freed, unit, out);
mi_print_amount(unit, 1, out);
mi_print_count(stat->allocated, unit, out);
mi_print_amount(stat->peak, unit, out, arg);
mi_print_amount(stat->allocated, unit, out, arg);
mi_print_amount(stat->freed, unit, out, arg);
mi_print_amount(unit, 1, out, arg);
mi_print_count(stat->allocated, unit, out, arg);
if (stat->allocated > stat->freed)
_mi_fprintf(out, " not all freed!\n");
_mi_fprintf(out, arg, " not all freed!\n");
else
_mi_fprintf(out, " ok\n");
_mi_fprintf(out, arg, " ok\n");
}
else if (unit<0) {
mi_print_amount(stat->peak, -1, out);
mi_print_amount(stat->allocated, -1, out);
mi_print_amount(stat->freed, -1, out);
mi_print_amount(stat->peak, -1, out, arg);
mi_print_amount(stat->allocated, -1, out, arg);
mi_print_amount(stat->freed, -1, out, arg);
if (unit==-1) {
_mi_fprintf(out, "%22s", "");
_mi_fprintf(out, arg, "%22s", "");
}
else {
mi_print_amount(-unit, 1, out);
mi_print_count((stat->allocated / -unit), 0, out);
mi_print_amount(-unit, 1, out, arg);
mi_print_count((stat->allocated / -unit), 0, out, arg);
}
if (stat->allocated > stat->freed)
_mi_fprintf(out, " not all freed!\n");
_mi_fprintf(out, arg, " not all freed!\n");
else
_mi_fprintf(out, " ok\n");
_mi_fprintf(out, arg, " ok\n");
}
else {
mi_print_amount(stat->peak, 1, out);
mi_print_amount(stat->allocated, 1, out);
_mi_fprintf(out, "\n");
mi_print_amount(stat->peak, 1, out, arg);
mi_print_amount(stat->allocated, 1, out, arg);
_mi_fprintf(out, arg, "\n");
}
}
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out ) {
_mi_fprintf(out, "%10s:", msg);
mi_print_amount(stat->total, -1, out);
_mi_fprintf(out, "\n");
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
_mi_fprintf(out, arg, "%10s:", msg);
mi_print_amount(stat->total, -1, out, arg);
_mi_fprintf(out, arg, "\n");
}
static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out) {
static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) {
const int64_t avg_tens = (stat->count == 0 ? 0 : (stat->total*10 / stat->count));
const long avg_whole = (long)(avg_tens/10);
const long avg_frac1 = (long)(avg_tens%10);
_mi_fprintf(out, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1);
_mi_fprintf(out, arg, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1);
}
static void mi_print_header(mi_output_fun* out ) {
_mi_fprintf(out,"%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count ");
static void mi_print_header(mi_output_fun* out, void* arg ) {
_mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count ");
}
#if MI_STAT>1
static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out) {
static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) {
bool found = false;
char buf[64];
for (size_t i = 0; i <= max; i++) {
@ -224,14 +224,14 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin
int64_t unit = _mi_bin_size((uint8_t)i);
snprintf(buf, 64, "%s %3zu", fmt, i);
mi_stat_add(all, &bins[i], unit);
mi_stat_print(&bins[i], buf, unit, out);
mi_stat_print(&bins[i], buf, unit, out, arg);
}
}
//snprintf(buf, 64, "%s all", fmt);
//mi_stat_print(all, buf, 1);
if (found) {
_mi_fprintf(out, "\n");
mi_print_header(out);
_mi_fprintf(out, arg, "\n");
mi_print_header(out, arg);
}
}
#endif
@ -239,40 +239,40 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin
static void mi_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit);
static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out) mi_attr_noexcept {
mi_print_header(out);
static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out, void* arg) mi_attr_noexcept {
mi_print_header(out,arg);
#if MI_STAT>1
mi_stat_count_t normal = { 0,0,0,0 };
mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out);
mi_stat_print(&normal, "normal", 1, out);
mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out);
mi_stat_print(&stats->giant, "giant", (stats->giant_count.count == 0 ? 1 : -(stats->giant.allocated / stats->giant_count.count)), out);
mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out,arg);
mi_stat_print(&normal, "normal", 1, out, arg);
mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg);
mi_stat_print(&stats->giant, "giant", (stats->giant_count.count == 0 ? 1 : -(stats->giant.allocated / stats->giant_count.count)), out, arg);
mi_stat_count_t total = { 0,0,0,0 };
mi_stat_add(&total, &normal, 1);
mi_stat_add(&total, &stats->huge, 1);
mi_stat_add(&total, &stats->giant, 1);
mi_stat_print(&total, "total", 1, out);
_mi_fprintf(out, "malloc requested: ");
mi_print_amount(stats->malloc.allocated, 1, out);
_mi_fprintf(out, "\n\n");
mi_stat_print(&total, "total", 1, out, arg);
_mi_fprintf(out, arg, "malloc requested: ");
mi_print_amount(stats->malloc.allocated, 1, out, arg);
_mi_fprintf(out, arg, "\n\n");
#endif
mi_stat_print(&stats->reserved, "reserved", 1, out);
mi_stat_print(&stats->committed, "committed", 1, out);
mi_stat_print(&stats->reset, "reset", 1, out);
mi_stat_print(&stats->page_committed, "touched", 1, out);
mi_stat_print(&stats->segments, "segments", -1, out);
mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out);
mi_stat_print(&stats->segments_cache, "-cached", -1, out);
mi_stat_print(&stats->pages, "pages", -1, out);
mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out);
mi_stat_counter_print(&stats->pages_extended, "-extended", out);
mi_stat_counter_print(&stats->page_no_retire, "-noretire", out);
mi_stat_counter_print(&stats->mmap_calls, "mmaps", out);
mi_stat_counter_print(&stats->commit_calls, "commits", out);
mi_stat_print(&stats->threads, "threads", -1, out);
mi_stat_counter_print_avg(&stats->searches, "searches", out);
_mi_fprintf(out, "%10s: %7i\n", "numa nodes", _mi_os_numa_node_count());
if (elapsed > 0) _mi_fprintf(out, "%10s: %7ld.%03ld s\n", "elapsed", elapsed/1000, elapsed%1000);
mi_stat_print(&stats->reserved, "reserved", 1, out, arg);
mi_stat_print(&stats->committed, "committed", 1, out, arg);
mi_stat_print(&stats->reset, "reset", 1, out, arg);
mi_stat_print(&stats->page_committed, "touched", 1, out, arg);
mi_stat_print(&stats->segments, "segments", -1, out, arg);
mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg);
mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg);
mi_stat_print(&stats->pages, "pages", -1, out, arg);
mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg);
mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg);
mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg);
mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg);
mi_stat_counter_print(&stats->commit_calls, "commits", out, arg);
mi_stat_print(&stats->threads, "threads", -1, out, arg);
mi_stat_counter_print_avg(&stats->searches, "searches", out, arg);
_mi_fprintf(out, arg, "%10s: %7i\n", "numa nodes", _mi_os_numa_node_count());
if (elapsed > 0) _mi_fprintf(out, arg, "%10s: %7ld.%03ld s\n", "elapsed", elapsed/1000, elapsed%1000);
mi_msecs_t user_time;
mi_msecs_t sys_time;
@ -281,13 +281,13 @@ static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun
size_t page_reclaim;
size_t peak_commit;
mi_process_info(&user_time, &sys_time, &peak_rss, &page_faults, &page_reclaim, &peak_commit);
_mi_fprintf(out,"%10s: user: %ld.%03ld s, system: %ld.%03ld s, faults: %lu, reclaims: %lu, rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, (unsigned long)page_faults, (unsigned long)page_reclaim );
mi_printf_amount((int64_t)peak_rss, 1, out, "%s");
_mi_fprintf(out, arg, "%10s: user: %ld.%03ld s, system: %ld.%03ld s, faults: %lu, reclaims: %lu, rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, (unsigned long)page_faults, (unsigned long)page_reclaim );
mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s");
if (peak_commit > 0) {
_mi_fprintf(out,", commit charge: ");
mi_printf_amount((int64_t)peak_commit, 1, out, "%s");
_mi_fprintf(out, arg, ", commit charge: ");
mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");
}
_mi_fprintf(out,"\n");
_mi_fprintf(out, arg, "\n");
}
static mi_msecs_t mi_time_start; // = 0
@ -319,20 +319,20 @@ void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done`
mi_stats_merge_from(stats);
}
static void mi_stats_print_ex(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out) {
mi_stats_merge_from(stats);
_mi_stats_print(&_mi_stats_main, elapsed, out);
void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
mi_msecs_t elapsed = _mi_clock_end(mi_time_start);
mi_stats_merge_from(mi_stats_get_default());
_mi_stats_print(&_mi_stats_main, elapsed, out, arg);
}
void mi_stats_print(mi_output_fun* out) mi_attr_noexcept {
mi_msecs_t elapsed = _mi_clock_end(mi_time_start);
mi_stats_print_ex(mi_stats_get_default(),elapsed,out);
void mi_stats_print(void* out) mi_attr_noexcept {
// for compatibility there is an `out` parameter (which can be `stdout` or `stderr`)
mi_stats_print_out((mi_output_fun*)out, NULL);
}
void mi_thread_stats_print(mi_output_fun* out) mi_attr_noexcept {
void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
mi_msecs_t elapsed = _mi_clock_end(mi_time_start);
_mi_stats_print(mi_stats_get_default(), elapsed, out);
_mi_stats_print(mi_stats_get_default(), elapsed, out, arg);
}

2
test/CMakeLists.txt
View File

@ -13,7 +13,7 @@ if (NOT CMAKE_BUILD_TYPE)
endif()
# Import mimalloc (if installed)
find_package(mimalloc 1.3 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH)
find_package(mimalloc 1.4 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH)
message(STATUS "Found mimalloc installed at: ${MIMALLOC_TARGET_DIR}")
# overriding with a dynamic library

4
test/main-override-static.c
View File

@ -13,7 +13,7 @@ static void corrupt_free();
int main() {
mi_version();
// detect double frees and heap corruption
// double_free1();
// double_free2();
@ -106,4 +106,4 @@ static void corrupt_free() {
for (int i = 0; i < 4096; i++) {
malloc(SZ);
}
}
}

4
test/test-stress.c
View File

@ -119,7 +119,7 @@ static void free_items(void* p) {
static void stress(intptr_t tid) {
//bench_start_thread();
uintptr_t r = tid * 43;
const size_t max_item_shift = 5; // 128
const size_t max_item_shift = 5; // 128
const size_t max_item_retained_shift = max_item_shift + 2;
size_t allocs = 100 * ((size_t)SCALE) * (tid % 8 + 1); // some threads do more
size_t retain = allocs / 2;
@ -135,7 +135,7 @@ static void stress(intptr_t tid) {
allocs--;
if (data_top >= data_size) {
data_size += 100000;
data = (void**)custom_realloc(data, data_size * sizeof(void*));
data = (void**)custom_realloc(data, data_size * sizeof(void*));
}
data[data_top++] = alloc_items(1ULL << (pick(&r) % max_item_shift), &r);
}