wip: new segment allocation

This commit is contained in:
daan 2019-08-15 11:49:56 -07:00
parent bbd81bbbd1
commit f2bafbc57f
10 changed files with 116 additions and 79 deletions

View File

@ -16,7 +16,6 @@ set(mi_install_dir "lib/mimalloc-${mi_version}")
set(mi_sources
src/stats.c
src/os.c
src/memory.c
src/segment.c
src/page.c
src/alloc.c

View File

@ -271,7 +271,7 @@ static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) {
// Get the page containing the pointer
static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) {
ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment;
mi_assert_internal(diff >= 0 && diff < MI_SEGMENT_SIZE);
mi_assert_internal(diff >= 0 && diff < (ptrdiff_t)MI_SEGMENT_SIZE);
uintptr_t idx = (uintptr_t)diff >> MI_SEGMENT_SLICE_SHIFT;
mi_assert_internal(idx < segment->slice_count);
mi_slice_t* slice0 = (mi_slice_t*)&segment->slices[idx];

View File

@ -375,7 +375,7 @@ typedef struct mi_segment_queue_s {
mi_segment_t* last;
} mi_segment_queue_t;
#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SEGMENT_SIZE)
#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT)
// Segments thread local data
typedef struct mi_segments_tld_s {

View File

@ -255,7 +255,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
}
#if (MI_STAT>1)
size_t inuse = page->used - page->thread_freed;
if (page->block_size <= MI_MEDIUM_SIZE_MAX) {
if (page->block_size <= MI_LARGE_SIZE_MAX) {
mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse);
}
mi_heap_stat_decrease(heap,malloc, page->block_size * inuse); // todo: off for aligned blocks...

View File

@ -92,13 +92,14 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
// Empty page queues for every bin
#define SQNULL(sz) { NULL, NULL, sz }
#define MI_SEGMENT_PAGE_QUEUES_EMPTY \
{ QNULL(0), \
QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \
QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \
QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \
QNULL( 160), QNULL( 192), QNULL( 224), /* 27 */ }
{ SQNULL(1), \
SQNULL( 1), SQNULL( 2), SQNULL( 3), SQNULL( 4), SQNULL( 5), SQNULL( 6), SQNULL( 7), SQNULL( 10), /* 8 */ \
SQNULL( 12), SQNULL( 14), SQNULL( 16), SQNULL( 20), SQNULL( 24), SQNULL( 28), SQNULL( 32), SQNULL( 40), /* 16 */ \
SQNULL( 48), SQNULL( 56), SQNULL( 64), SQNULL( 80), SQNULL( 96), SQNULL( 112), SQNULL( 128), SQNULL( 160), /* 24 */ \
SQNULL( 192), SQNULL( 224), SQNULL( 256), SQNULL( 320), SQNULL( 384), SQNULL( 448), SQNULL( 512), SQNULL( 640), /* 32 */ \
SQNULL( 768), SQNULL( 896), SQNULL( 1024) /* 35 */ }
#define tld_main_stats ((mi_stats_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,stats)))

View File

@ -99,7 +99,7 @@ bool _mi_page_is_valid(mi_page_t* page) {
#endif
if (page->heap!=NULL) {
mi_segment_t* segment = _mi_page_segment(page);
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id);
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == page->heap->thread_id);
mi_page_queue_t* pq = mi_page_queue_of(page);
mi_assert_internal(mi_page_queue_contains(pq, page));
mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_MEDIUM_SIZE_MAX || mi_page_is_in_full(page));

View File

@ -13,6 +13,9 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_PAGE_HUGE_ALIGN (256*1024)
static void mi_segment_map_allocated_at(const mi_segment_t* segment);
static void mi_segment_map_freed_at(const mi_segment_t* segment);
/* -----------------------------------------------------------
Segment allocation
@ -50,21 +53,13 @@ static inline size_t mi_bsr(uintptr_t x) {
#error "define bsr for your platform"
#endif
static size_t mi_slice_bin4(size_t slice_count) {
if (slice_count==0) return 0;
mi_assert_internal(slice_count <= MI_SLICES_PER_SEGMENT);
size_t s = mi_bsr(slice_count);
if (s <= 1) return slice_count;
size_t bin = ((s << 1) | (slice_count >> (s - 1))&0x01);
return bin;
}
static size_t mi_slice_bin8(size_t slice_count) {
if (slice_count==0) return 0;
if (slice_count<=1) return slice_count;
mi_assert_internal(slice_count <= MI_SLICES_PER_SEGMENT);
slice_count--;
size_t s = mi_bsr(slice_count);
if (s <= 2) return slice_count;
size_t bin = ((s << 2) | (slice_count >> (s - 2))&0x03) - 5;
if (s <= 2) return slice_count + 1;
size_t bin = ((s << 2) | ((slice_count >> (s - 2))&0x03)) - 4;
return bin;
}
@ -72,7 +67,7 @@ static size_t mi_slice_bin(size_t slice_count) {
mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_SEGMENT_SIZE);
mi_assert_internal(mi_slice_bin8(MI_SLICES_PER_SEGMENT) == MI_SEGMENT_BIN_MAX);
size_t bin = (slice_count==0 ? 0 : mi_slice_bin8(slice_count));
mi_assert_internal(bin >= 0 && bin <= MI_SEGMENT_BIN_MAX);
mi_assert_internal(bin <= MI_SEGMENT_BIN_MAX);
return bin;
}
@ -80,6 +75,7 @@ static size_t mi_slice_bin(size_t slice_count) {
/* -----------------------------------------------------------
Page Queues
----------------------------------------------------------- */
/*
static bool mi_page_queue_is_empty(mi_page_queue_t* pq) {
return (pq->first == NULL);
}
@ -97,6 +93,7 @@ static mi_page_t* mi_page_queue_pop(mi_page_queue_t* pq)
page->block_size = 1; // no more free
return page;
}
*/
static void mi_page_queue_push(mi_page_queue_t* pq, mi_page_t* page) {
// todo: or push to the end?
@ -111,15 +108,18 @@ static void mi_page_queue_push(mi_page_queue_t* pq, mi_page_t* page) {
static mi_page_queue_t* mi_page_queue_for(size_t slice_count, mi_segments_tld_t* tld) {
size_t bin = mi_slice_bin(slice_count);
return &tld->pages[bin];
mi_page_queue_t* pq = &tld->pages[bin];
// mi_assert_internal(pq->block_size >= slice_count);
return pq;
}
static void mi_page_queue_remove(mi_page_queue_t* pq, mi_page_t* page) {
static void mi_page_queue_delete(mi_page_queue_t* pq, mi_page_t* page) {
mi_assert_internal(page->block_size==0 && page->slice_count>0 && page->slice_offset==0);
// should work too if the queue does not contain page (which can happen during reclaim)
if (page->prev != NULL) page->prev->next = page->next;
else pq->first = page->next;
if (page == pq->first) pq->first = page->next;
if (page->next != NULL) page->next->prev = page->prev;
else pq->last = page->prev;
if (page == pq->last) pq->last = page->prev;
page->prev = NULL;
page->next = NULL;
page->block_size = 1; // no more free
@ -145,13 +145,13 @@ static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id());
//mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0);
mi_slice_t* slice = &segment->slices[0];
size_t page_count = 0;
size_t used_count = 0;
mi_page_queue_t* pq;
while(slice < &segment->slices[segment->slice_count]) {
mi_assert_internal(slice->slice_count > 0);
mi_assert_internal(slice->slice_offset == 0);
if (slice->block_size > 0) { // a page in use, all slices need their back offset set
page_count++;
used_count++;
for (size_t i = 0; i < slice->slice_count; i++) {
mi_assert_internal((slice+i)->slice_offset == i);
mi_assert_internal(i==0 || (slice+i)->slice_count == 0);
@ -171,7 +171,7 @@ static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
slice = slice + slice->slice_count;
}
mi_assert_internal(slice == &segment->slices[segment->slice_count]);
mi_assert_internal(page_count == segment->used + 1);
mi_assert_internal(used_count == segment->used + 1);
return true;
}
#endif
@ -255,6 +255,7 @@ static void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) {
static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_segments_tld_t* tld) {
segment->thread_id = 0;
mi_segment_map_freed_at(segment);
mi_segments_track_size(-((long)segment_size),tld);
if (mi_option_is_enabled(mi_option_secure)) {
_mi_os_unprotect(segment, segment->segment_size); // ensure no more guard pages are set
@ -265,7 +266,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
// The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use,
// and no more than 1.
#define MI_SEGMENT_CACHE_MAX (1)
#define MI_SEGMENT_CACHE_MAX (2)
#define MI_SEGMENT_CACHE_FRACTION (8)
// note: returned segment may be partially reset
@ -344,10 +345,10 @@ static mi_slice_t* mi_segment_last_slice(mi_segment_t* segment) {
static void mi_segment_page_init(mi_segment_t* segment, size_t slice_index, size_t slice_count, mi_segments_tld_t* tld) {
mi_assert_internal(slice_index >= 0 && slice_index < segment->slice_count);
size_t bin = mi_slice_bin(slice_count);
mi_assert_internal(slice_index < segment->slice_count);
mi_page_queue_t* pq = mi_page_queue_for(slice_count,tld);
if (slice_count==0) slice_count = 1;
mi_assert_internal(slice_count >= 0 && slice_index + slice_count - 1 < segment->slice_count);
mi_assert_internal(slice_index + slice_count - 1 < segment->slice_count);
// set first and last slice (the intermediates can be undetermined)
mi_slice_t* slice = &segment->slices[slice_index];
@ -360,7 +361,7 @@ static void mi_segment_page_init(mi_segment_t* segment, size_t slice_index, size
end->block_size = 0;
}
// and push it on the free page queue
mi_page_queue_push( &tld->pages[bin], mi_slice_to_page(slice) );
mi_page_queue_push( pq, mi_slice_to_page(slice) );
}
static void mi_segment_page_add_free(mi_page_t* page, mi_segments_tld_t* tld) {
@ -368,6 +369,7 @@ static void mi_segment_page_add_free(mi_page_t* page, mi_segments_tld_t* tld) {
mi_assert_internal(page->block_size==0 && page->slice_count>0 && page->slice_offset==0);
size_t slice_index = mi_slice_index(mi_page_to_slice(page));
mi_segment_page_init(segment,slice_index,page->slice_count,tld);
}
@ -386,28 +388,28 @@ static mi_page_t* mi_segment_page_find(size_t slice_count, mi_segments_tld_t* tl
// search from best fit up
mi_page_queue_t* pq = mi_page_queue_for(slice_count,tld);
if (slice_count == 0) slice_count = 1;
while (pq <= &tld->pages[MI_SEGMENT_BIN_MAX] && mi_page_queue_is_empty(pq)) {
pq++;
}
if (pq > &tld->pages[MI_SEGMENT_BIN_MAX]) {
// could not find a page..
return NULL;
}
// pop the page and split to the right size
mi_page_t* page = mi_page_queue_pop(pq);
mi_assert_internal(page != NULL && page->slice_count >= slice_count && page->slice_offset == 0);
while (pq <= &tld->pages[MI_SEGMENT_BIN_MAX]) {
for( mi_page_t* page = pq->first; page != NULL; page = page->next) {
if (page->slice_count >= slice_count) {
// found one
mi_page_queue_delete(pq,page);
if (page->slice_count > slice_count) {
mi_segment_page_split(page,slice_count,tld);
}
mi_assert_internal(page != NULL && page->slice_count == slice_count);
return page;
}
}
pq++;
}
// could not find a page..
return NULL;
}
static void mi_segment_page_remove(mi_slice_t* slice, mi_segments_tld_t* tld) {
static void mi_segment_page_delete(mi_slice_t* slice, mi_segments_tld_t* tld) {
mi_assert_internal(slice->slice_count > 0 && slice->slice_offset==0 && slice->block_size==0);
mi_page_queue_t* pq = mi_page_queue_for(slice->slice_count, tld);
mi_page_queue_remove(pq, mi_slice_to_page(slice));
mi_page_queue_delete(pq, mi_slice_to_page(slice));
}
@ -440,6 +442,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_segments_tld_t* tld, m
}
segment->memid = memid;
mi_segments_track_size((long)segment_size, tld);
mi_segment_map_allocated_at(segment);
}
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
@ -506,7 +509,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t
mi_assert_internal(slice->slice_offset == 0);
mi_assert_internal(mi_slice_index(slice)==0 || slice->block_size == 0); // no more used pages ..
if (slice->block_size == 0) {
mi_segment_page_remove(slice, tld);
mi_segment_page_delete(slice, tld);
}
page_count++;
slice = slice + slice->slice_count;
@ -573,7 +576,7 @@ static mi_page_t* mi_segment_page_alloc(mi_page_kind_t page_kind, size_t require
return page;
}
static void mi_segment_page_free_coalesce(mi_page_t* page, mi_segments_tld_t* tld) {
static mi_slice_t* mi_segment_page_free_coalesce(mi_page_t* page, mi_segments_tld_t* tld) {
mi_assert_internal(page != NULL && page->slice_count > 0 && page->slice_offset == 0 && page->block_size > 0);
mi_segment_t* segment = _mi_page_segment(page);
mi_assert_internal(segment->used > 0);
@ -588,7 +591,7 @@ static void mi_segment_page_free_coalesce(mi_page_t* page, mi_segments_tld_t* tl
// free next block -- remove it from free and merge
mi_assert_internal(next->slice_count > 0 && next->slice_offset==0);
slice_count += next->slice_count; // extend
mi_segment_page_remove(next, tld);
mi_segment_page_delete(next, tld);
}
if (slice > segment->slices) {
mi_slice_t* prev = slice - 1;
@ -598,7 +601,7 @@ static void mi_segment_page_free_coalesce(mi_page_t* page, mi_segments_tld_t* tl
// free previous slice -- remove it from free and merge
mi_assert_internal(prev->slice_count > 0 && prev->slice_offset==0);
slice_count += prev->slice_count;
mi_segment_page_remove(prev, tld);
mi_segment_page_delete(prev, tld);
slice = prev;
}
}
@ -606,6 +609,7 @@ static void mi_segment_page_free_coalesce(mi_page_t* page, mi_segments_tld_t* tl
// and add the new free page
mi_segment_page_init(segment, mi_slice_index(slice), slice_count, tld);
mi_assert_expensive(mi_segment_is_valid(segment,tld));
return slice;
}
@ -615,7 +619,7 @@ static void mi_segment_page_free_coalesce(mi_page_t* page, mi_segments_tld_t* tl
static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld);
static void mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) {
static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) {
mi_assert_internal(page->block_size > 0);
mi_assert_internal(mi_page_all_free(page));
mi_segment_t* segment = _mi_ptr_segment(page);
@ -643,7 +647,7 @@ static void mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) {
page->block_size = 1;
// and free it
mi_segment_page_free_coalesce(page, tld);
return mi_segment_page_free_coalesce(page, tld);
}
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)
@ -683,7 +687,19 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(segment->abandoned_next == NULL);
mi_assert_expensive(mi_segment_is_valid(segment,tld));
// all pages in the segment are abandoned; add it to the abandoned list
// remove the free pages from our lists
mi_slice_t* slice = &segment->slices[0];
while (slice < mi_segment_last_slice(segment)) {
mi_assert_internal(slice->slice_count > 0);
mi_assert_internal(slice->slice_offset == 0);
if (slice->block_size == 0) { // a free page
mi_segment_page_delete(slice,tld);
slice->block_size = 0; // but keep it free
}
slice = slice + slice->slice_count;
}
// add it to the abandoned list
segment->thread_id = 0;
do {
segment->abandoned_next = (mi_segment_t*)abandoned;
@ -730,23 +746,38 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
mi_atomic_decrement(&abandoned_count);
mi_assert_expensive(mi_segment_is_valid(segment, tld));
segment->abandoned_next = NULL;
segment->thread_id = _mi_thread_id();
mi_segments_track_size((long)segment->segment_size,tld);
mi_assert_internal(segment->next == NULL && segment->prev == NULL);
_mi_stat_decrease(&tld->stats->segments_abandoned,1);
mi_slice_t* slice = &segment->slices[0];
while (slice < mi_segment_last_slice(segment)) {
mi_assert_internal(slice->slice_count>0 && slice->block_size>0); // segment allocated page
slice = slice + slice->slice_count; // skip the first segment allocated page
while (slice <= mi_segment_last_slice(segment)) {
mi_assert_internal(slice->slice_count > 0);
mi_assert_internal(slice->slice_offset == 0);
mi_page_t* page = mi_slice_to_page(slice);
if (page->block_size == 0) { // a free page, add it to our lists
mi_segment_page_add_free(page,tld);
}
slice = slice + slice->slice_count;
if (page->block_size > 0) { // a page in use
segment->abandoned--;
}
slice = &segment->slices[0];
mi_assert_internal(slice->slice_count>0 && slice->block_size>0); // segment allocated page
slice = slice + slice->slice_count; // skip the first segment allocated page
while (slice <= mi_segment_last_slice(segment)) {
mi_assert_internal(slice->slice_count > 0);
mi_assert_internal(slice->slice_offset == 0);
mi_page_t* page = mi_slice_to_page(slice);
if (page->block_size > 0) { // a used page
mi_assert_internal(page->next == NULL && page->prev==NULL);
_mi_stat_decrease(&tld->stats->pages_abandoned, 1);
segment->abandoned--;
if (mi_page_all_free(page)) {
// if everything free by now, free the page
mi_segment_page_clear(page, tld);
slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing
}
else {
// otherwise reclaim it
@ -754,13 +785,11 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen
_mi_page_reclaim(heap, page);
}
}
else { // free range of slices; add to the free pages
mi_segment_page_add_free(page,tld);
}
mi_assert_internal(slice->slice_count>0 && slice->slice_offset==0);
slice = slice + slice->slice_count;
}
mi_assert(segment->abandoned == 0);
segment->thread_id = _mi_thread_id(); // only now for valid checks
if (segment->used == 0) { // due to page_clear
mi_segment_free(segment,false,tld);
}
@ -917,6 +946,11 @@ static bool mi_is_valid_pointer(const void* p) {
return (_mi_segment_of(p) != NULL);
}
bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
return mi_is_valid_pointer(p);
}
/*
// Return the full segment range belonging to a pointer
static void* mi_segment_range_of(const void* p, size_t* size) {
mi_segment_t* segment = _mi_segment_of(p);
@ -929,8 +963,6 @@ static void* mi_segment_range_of(const void* p, size_t* size) {
return segment;
}
}
*/
bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
return mi_is_valid_pointer(p);
}

View File

@ -15,7 +15,7 @@ terms of the MIT license. A copy of the license can be found in the file
// functions (on Unix's).
#include "stats.c"
#include "os.c"
#include "memory.c"
//#include "memory.c"
#include "segment.c"
#include "page.c"
#include "heap.c"

View File

@ -231,8 +231,8 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n
mi_stat_count_t normal = { 0,0,0,0 };
mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out);
mi_stat_print(&normal, "normal", 1, out);
mi_stat_print(&stats->large, "large", (stats->large_count.count == 0 ? 1 : -(stats->large.allocated / stats->large_count.count)), out);
mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out);
mi_stat_print(&stats->large, "giant", (stats->large_count.count == 0 ? 1 : -(stats->large.allocated / stats->large_count.count)), out);
mi_stat_count_t total = { 0,0,0,0 };
mi_stat_add(&total, &normal, 1);
mi_stat_add(&total, &stats->huge, 1);

View File

@ -143,7 +143,7 @@ size_t _mi_binx4(size_t bsize) {
}
size_t _mi_binx8(size_t bsize) {
if (bsize==0) return 0;
if (bsize<=1) return bsize;
uint8_t b = mi_bsr32((uint32_t)bsize);
if (b <= 2) return bsize;
size_t bin = ((b << 2) | (bsize >> (b - 2))&0x03) - 5;
@ -152,16 +152,20 @@ size_t _mi_binx8(size_t bsize) {
void mi_bins() {
//printf(" QNULL(1), /* 0 */ \\\n ");
size_t last_bin = 1;
for (size_t bsize = 0; bsize < 8*1024; bsize++) {
size_t last_bin = 0;
size_t min_bsize = 0;
size_t last_bsize = 0;
for (size_t bsize = 1; bsize < 2*1024; bsize++) {
size_t size = bsize * 64 * 1024;
size_t bin = _mi_binx8(bsize);
if (bin != last_bin) {
printf("bsize: %6zd, size: %6zd, bin: %6zd\n", bsize, size, bin);
printf("min bsize: %6zd, max bsize: %6zd, bin: %6zd\n", min_bsize, last_bsize, last_bin);
//printf("QNULL(%6zd), ", wsize);
//if (last_bin%8 == 0) printf("/* %i */ \\\n ", last_bin);
last_bin = bin;
min_bsize = bsize;
}
last_bsize = bsize;
}
}
@ -186,6 +190,7 @@ int main() {
//free(p1);
//p2 = malloc(32);
//mi_free(p2);
mi_collect(true);
mi_stats_print(NULL);
return 0;
}