mirrors/mimalloc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve segment_cache assertions

Browse Source
This commit is contained in:
Daan Leijen 2023-03-23 16:11:38 -07:00
parent 1cbc55f2b8
commit 165b847051
6 changed files with 48 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/mimalloc/internal.h
View File

@ -115,7 +115,7 @@ mi_arena_id_t _mi_arena_id_none(void);
bool _mi_arena_memid_is_suitable(size_t memid, mi_arena_id_t req_arena_id); bool _mi_arena_memid_is_suitable(size_t memid, mi_arena_id_t req_arena_id);
// "segment-cache.c" // "segment-cache.c"
void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld); void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool large_allowed, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld);
bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld); bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld);
void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld); void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld);
void _mi_segment_cache_free_all(mi_os_tld_t* tld); void _mi_segment_cache_free_all(mi_os_tld_t* tld);

2
src/os.c
View File

@ -364,7 +364,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
int err = _mi_prim_commit(start, csize, commit); int err = _mi_prim_commit(start, csize, commit);
if (err != 0) { if (err != 0) {
_mi_warning_message("cannot %s OS memory (error: %d (0x%d), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize); _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", commit ? "commit" : "decommit", err, err, start, csize);
} }
mi_assert_internal(err == 0); mi_assert_internal(err == 0);
return (err == 0); return (err == 0);

2
src/page.c
View File

@ -92,8 +92,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
} }
#endif #endif
#if !MI_TSAN
mi_block_t* tfree = mi_page_thread_free(page); mi_block_t* tfree = mi_page_thread_free(page);
mi_assert_internal(mi_page_list_is_valid(page, tfree)); mi_assert_internal(mi_page_list_is_valid(page, tfree));
#endif
//size_t tfree_count = mi_page_list_count(page, tfree); //size_t tfree_count = mi_page_list_count(page, tfree);
//mi_assert_internal(tfree_count <= page->thread_freed + 1); //mi_assert_internal(tfree_count <= page->thread_freed + 1);

2
src/prim/unix/prim.c
View File

@ -395,7 +395,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, vo
long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
if (err != 0) { if (err != 0) {
err = errno; err = errno;
_mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%d))\n", numa_node, err, err); _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);
} }
} }
return (*addr != NULL ? 0 : errno); return (*addr != NULL ? 0 : errno);

77
src/segment-cache.c
View File

@ -35,8 +35,8 @@ typedef struct mi_cache_slot_s {
static mi_decl_cache_align mi_cache_slot_t cache[MI_CACHE_MAX]; // = 0 static mi_decl_cache_align mi_cache_slot_t cache[MI_CACHE_MAX]; // = 0
static mi_decl_cache_align mi_bitmap_field_t cache_available[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; // zero bit = available! static mi_decl_cache_align mi_bitmap_field_t cache_unavailable[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; // zero bit = available!
static mi_decl_cache_align mi_bitmap_field_t cache_available_large[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET }; static mi_decl_cache_align mi_bitmap_field_t cache_unavailable_large[MI_CACHE_FIELDS] = { MI_CACHE_BITS_SET };
static mi_decl_cache_align mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free static mi_decl_cache_align mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free
static bool mi_cdecl mi_segment_cache_is_suitable(mi_bitmap_index_t bitidx, void* arg) { static bool mi_cdecl mi_segment_cache_is_suitable(mi_bitmap_index_t bitidx, void* arg) {
@ -48,7 +48,8 @@ static bool mi_cdecl mi_segment_cache_is_suitable(mi_bitmap_index_t bitidx, void
mi_decl_noinline static void* mi_segment_cache_pop_ex( mi_decl_noinline static void* mi_segment_cache_pop_ex(
bool all_suitable, bool all_suitable,
size_t size, mi_commit_mask_t* commit_mask, size_t size, mi_commit_mask_t* commit_mask,
mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_commit_mask_t* decommit_mask, bool large_allowed,
bool* large, bool* is_pinned, bool* is_zero,
mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld) mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld)
{ {
#ifdef MI_CACHE_DISABLE #ifdef MI_CACHE_DISABLE
@ -66,23 +67,28 @@ mi_decl_noinline static void* mi_segment_cache_pop_ex(
if (start_field >= MI_CACHE_FIELDS) start_field = 0; if (start_field >= MI_CACHE_FIELDS) start_field = 0;
} }
// find an available slot // find an available slot and make it unavailable
mi_bitmap_index_t bitidx = 0; mi_bitmap_index_t bitidx = 0;
bool claimed = false; bool claimed = false;
mi_arena_id_t req_arena_id = _req_arena_id; mi_arena_id_t req_arena_id = _req_arena_id;
mi_bitmap_pred_fun_t pred_fun = (all_suitable ? NULL : &mi_segment_cache_is_suitable); // cannot pass NULL as the arena may be exclusive itself; todo: do not put exclusive arenas in the cache? mi_bitmap_pred_fun_t pred_fun = (all_suitable ? NULL : &mi_segment_cache_is_suitable); // cannot pass NULL as the arena may be exclusive itself; todo: do not put exclusive arenas in the cache?
if (*large) { // large allowed? if (large_allowed) { // large allowed?
claimed = _mi_bitmap_try_find_from_claim_pred(cache_available_large, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); claimed = _mi_bitmap_try_find_from_claim_pred(cache_unavailable_large, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx);
if (claimed) *large = true; if (claimed) *large = true;
} }
if (!claimed) { if (!claimed) {
claimed = _mi_bitmap_try_find_from_claim_pred (cache_available, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx); claimed = _mi_bitmap_try_find_from_claim_pred (cache_unavailable, MI_CACHE_FIELDS, start_field, 1, pred_fun, &req_arena_id, &bitidx);
if (claimed) *large = false; if (claimed) *large = false;
} }
if (!claimed) return NULL; if (!claimed) return NULL;
// no longer available but still in-use
mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx));
mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable_large, MI_CACHE_FIELDS, 1, bitidx));
mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx));
// found a slot // found a slot
mi_cache_slot_t* slot = &cache[mi_bitmap_index_bit(bitidx)]; mi_cache_slot_t* slot = &cache[mi_bitmap_index_bit(bitidx)];
void* p = slot->p; void* p = slot->p;
@ -95,16 +101,15 @@ mi_decl_noinline static void* mi_segment_cache_pop_ex(
mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0);
// mark the slot as free again // mark the slot as free again
mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx));
_mi_bitmap_unclaim(cache_inuse, MI_CACHE_FIELDS, 1, bitidx); _mi_bitmap_unclaim(cache_inuse, MI_CACHE_FIELDS, 1, bitidx);
return p; return p;
#endif #endif
} }
mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld) mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool large_allowed, bool* large, bool* is_pinned, bool* is_zero, mi_arena_id_t _req_arena_id, size_t* memid, mi_os_tld_t* tld)
{ {
return mi_segment_cache_pop_ex(false, size, commit_mask, decommit_mask, large, is_pinned, is_zero, _req_arena_id, memid, tld); return mi_segment_cache_pop_ex(false, size, commit_mask, decommit_mask, large_allowed, large, is_pinned, is_zero, _req_arena_id, memid, tld);
} }
static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats) static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats)
@ -113,10 +118,11 @@ static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, vo
// nothing // nothing
} }
else if (mi_commit_mask_is_full(cmask)) { else if (mi_commit_mask_is_full(cmask)) {
// decommit the whole in one call
_mi_os_decommit(p, total, stats); _mi_os_decommit(p, total, stats);
} }
else { else {
// todo: one call to decommit the whole at once? // decommit parts
mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0); mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
size_t part = total/MI_COMMIT_MASK_BITS; size_t part = total/MI_COMMIT_MASK_BITS;
size_t idx; size_t idx;
@ -148,21 +154,25 @@ static mi_decl_noinline void mi_segment_cache_purge(bool visit_all, bool force,
if (expire != 0 && (force || now >= expire)) { // racy read if (expire != 0 && (force || now >= expire)) { // racy read
// seems expired, first claim it from available // seems expired, first claim it from available
purged++; purged++;
mi_bitmap_index_t bitidx = mi_bitmap_index_create_from_bit(idx); mi_bitmap_index_t bitidx = mi_bitmap_index_create_from_bit(idx);
if (_mi_bitmap_claim(cache_available, MI_CACHE_FIELDS, 1, bitidx, NULL)) { if (_mi_bitmap_claim(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx, NULL)) { // no need to check large as those cannot be decommitted anyways
// was available, we claimed it // it was available, we claimed it (and made it unavailable)
mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx));
mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable_large, MI_CACHE_FIELDS, 1, bitidx));
// we can now access it safely
expire = mi_atomic_loadi64_acquire(&slot->expire); expire = mi_atomic_loadi64_acquire(&slot->expire);
if (expire != 0 && (force || now >= expire)) { // safe read if (expire != 0 && (force || now >= expire)) { // safe read
mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx));
// still expired, decommit it // still expired, decommit it
mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
mi_assert_internal(!mi_commit_mask_is_empty(&slot->commit_mask) && _mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(!mi_commit_mask_is_empty(&slot->commit_mask));
_mi_abandoned_await_readers(); // wait until safe to decommit _mi_abandoned_await_readers(); // wait until safe to decommit
// decommit committed parts // decommit committed parts
// TODO: instead of decommit, we could also free to the OS? // TODO: instead of decommit, we could also free to the OS?
mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats); mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats);
mi_commit_mask_create_empty(&slot->decommit_mask); mi_commit_mask_create_empty(&slot->decommit_mask);
} }
_mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop _mi_bitmap_unclaim(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop
} }
if (!visit_all && purged > MI_MAX_PURGE_PER_PUSH) break; // bound to no more than N purge tries per push if (!visit_all && purged > MI_MAX_PURGE_PER_PUSH) break; // bound to no more than N purge tries per push
} }
@ -184,23 +194,20 @@ void _mi_segment_cache_free_all(mi_os_tld_t* tld) {
mi_commit_mask_t decommit_mask; mi_commit_mask_t decommit_mask;
bool is_pinned; bool is_pinned;
bool is_zero; bool is_zero;
bool is_large;
size_t memid; size_t memid;
const size_t size = MI_SEGMENT_SIZE; const size_t size = MI_SEGMENT_SIZE;
// iterate twice: first large pages, then regular memory void* p;
for (int i = 0; i < 2; i++) { do {
void* p; // keep popping and freeing the memory
do { p = mi_segment_cache_pop_ex(true /* all */, size, &commit_mask, &decommit_mask,
// keep popping and freeing the memory true /* allow large */, &is_large, &is_pinned, &is_zero, _mi_arena_id_none(), &memid, tld);
bool large = (i == 0); if (p != NULL) {
p = mi_segment_cache_pop_ex(true /* all */, size, &commit_mask, &decommit_mask, size_t csize = _mi_commit_mask_committed_size(&commit_mask, size);
&large, &is_pinned, &is_zero, _mi_arena_id_none(), &memid, tld); if (csize > 0 && !is_pinned) { _mi_stat_decrease(&_mi_stats_main.committed, csize); }
if (p != NULL) { _mi_arena_free(p, size, MI_SEGMENT_ALIGN, 0, memid, is_pinned /* pretend not committed to not double count decommits */, tld->stats);
size_t csize = _mi_commit_mask_committed_size(&commit_mask, size); }
if (csize > 0 && !is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize); } while (p != NULL);
_mi_arena_free(p, size, MI_SEGMENT_ALIGN, 0, memid, is_pinned /* pretend not committed to not double count decommits */, tld->stats);
}
} while (p != NULL);
}
} }
mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld) mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld)
@ -228,8 +235,8 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me
bool claimed = _mi_bitmap_try_find_from_claim(cache_inuse, MI_CACHE_FIELDS, start_field, 1, &bitidx); bool claimed = _mi_bitmap_try_find_from_claim(cache_inuse, MI_CACHE_FIELDS, start_field, 1, &bitidx);
if (!claimed) return false; if (!claimed) return false;
mi_assert_internal(_mi_bitmap_is_claimed(cache_available, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable, MI_CACHE_FIELDS, 1, bitidx));
mi_assert_internal(_mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(_mi_bitmap_is_claimed(cache_unavailable_large, MI_CACHE_FIELDS, 1, bitidx));
#if MI_DEBUG>1 #if MI_DEBUG>1
if (is_pinned || is_large) { if (is_pinned || is_large) {
mi_assert_internal(mi_commit_mask_is_full(commit_mask)); mi_assert_internal(mi_commit_mask_is_full(commit_mask));
@ -257,7 +264,7 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me
} }
// make it available // make it available
_mi_bitmap_unclaim((is_large ? cache_available_large : cache_available), MI_CACHE_FIELDS, 1, bitidx); _mi_bitmap_unclaim((is_large ? cache_unavailable_large : cache_unavailable), MI_CACHE_FIELDS, 1, bitidx);
return true; return true;
#endif #endif
} }
@ -273,7 +280,7 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me
#if (MI_INTPTR_SIZE==8) #if (MI_INTPTR_SIZE==8)
#define MI_MAX_ADDRESS ((size_t)20 << 40) // 20TB #define MI_MAX_ADDRESS ((size_t)40 << 40) // 20TB
#else #else
#define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb #define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb
#endif #endif

2
src/segment.c
View File

@ -809,7 +809,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment
// get from cache? // get from cache?
if (page_alignment == 0) { if (page_alignment == 0) {
segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, pcommit_mask, pdecommit_mask, &mem_large, &is_pinned, is_zero, req_arena_id, &memid, os_tld); segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, pcommit_mask, pdecommit_mask, mem_large, &mem_large, &is_pinned, is_zero, req_arena_id, &memid, os_tld);
} }
// get from OS // get from OS