From 069c26dbab79ba11ba0ef2f272ae87769bdca68d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 16 Aug 2019 18:40:30 -0700 Subject: [PATCH] fix small object size check in mi_segment_page_alloc --- include/mimalloc-types.h | 11 ++++++----- src/alloc.c | 4 ++-- src/heap.c | 6 +++--- src/init.c | 4 ++-- src/page-queue.c | 18 +++++++++--------- src/page.c | 12 ++++++------ src/segment.c | 8 ++++---- 7 files changed, 32 insertions(+), 31 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 67ad8516..94188cc0 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -91,10 +91,11 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE) #define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) -#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit -#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1Mb on 64-bit -#define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT) -#define MI_HUGE_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) +#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) +#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit +#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1Mb on 64-bit +#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT) +#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) // Minimal alignment necessary. On most platforms 16 bytes are needed // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` @@ -103,7 +104,7 @@ terms of the MIT license. A copy of the license can be found in the file // Maximum number of size classes. (spaced exponentially in 12.5% increments) #define MI_BIN_HUGE (73U) -#if (MI_LARGE_WSIZE_MAX >= 655360) +#if (MI_LARGE_OBJ_WSIZE_MAX >= 655360) #error "define more bins" #endif diff --git a/src/alloc.c b/src/alloc.c index 7ad7c552..5cc172d3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -38,7 +38,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz block->next = 0; #endif #if (MI_STAT>1) - if(size <= MI_LARGE_SIZE_MAX) { + if(size <= MI_LARGE_OBJ_SIZE_MAX) { size_t bin = _mi_bin(size); mi_heap_stat_increase(heap,normal[bin], 1); } @@ -230,7 +230,7 @@ void mi_free(void* p) mi_attr_noexcept #if (MI_STAT>1) mi_heap_t* heap = mi_heap_get_default(); mi_heap_stat_decrease( heap, malloc, mi_usable_size(p)); - if (page->block_size <= MI_LARGE_SIZE_MAX) { + if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) { mi_heap_stat_decrease( heap, normal[_mi_bin(page->block_size)], 1); } // huge page stat is accounted for in `_mi_page_retire` diff --git a/src/heap.c b/src/heap.c index c18902b1..35c1d460 100644 --- a/src/heap.c +++ b/src/heap.c @@ -245,8 +245,8 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE); // stats - if (page->block_size > MI_LARGE_SIZE_MAX) { - if (page->block_size > MI_HUGE_SIZE_MAX) { + if (page->block_size > MI_LARGE_OBJ_SIZE_MAX) { + if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_decrease(&heap->tld->stats.giant,page->block_size); } else { @@ -255,7 +255,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ } #if (MI_STAT>1) size_t inuse = page->used - page->thread_freed; - if (page->block_size <= MI_LARGE_SIZE_MAX) { + if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) { mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse); } mi_heap_stat_decrease(heap,malloc, page->block_size * inuse); // todo: off for aligned blocks... diff --git a/src/init.c b/src/init.c index 77ce4aad..4fc5d602 100644 --- a/src/init.c +++ b/src/init.c @@ -43,8 +43,8 @@ const mi_page_t _mi_page_empty = { QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ - QNULL(MI_LARGE_WSIZE_MAX + 1 /* 655360, Huge queue */), \ - QNULL(MI_LARGE_WSIZE_MAX + 2) /* Full queue */ } + QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ + QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ } #define MI_STAT_COUNT_NULL() {0,0,0,0} diff --git a/src/page-queue.c b/src/page-queue.c index e59620c2..e476403b 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -34,15 +34,15 @@ terms of the MIT license. A copy of the license can be found in the file static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_LARGE_SIZE_MAX+sizeof(uintptr_t))); + return (pq->block_size == (MI_LARGE_OBJ_SIZE_MAX+sizeof(uintptr_t))); } static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_LARGE_SIZE_MAX+(2*sizeof(uintptr_t)))); + return (pq->block_size == (MI_LARGE_OBJ_SIZE_MAX+(2*sizeof(uintptr_t)))); } static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { - return (pq->block_size > MI_LARGE_SIZE_MAX); + return (pq->block_size > MI_LARGE_OBJ_SIZE_MAX); } /* ----------------------------------------------------------- @@ -116,7 +116,7 @@ extern inline uint8_t _mi_bin(size_t size) { bin = (uint8_t)wsize; } #endif - else if (wsize > MI_LARGE_WSIZE_MAX) { + else if (wsize > MI_LARGE_OBJ_WSIZE_MAX) { bin = MI_BIN_HUGE; } else { @@ -147,7 +147,7 @@ size_t _mi_bin_size(uint8_t bin) { // Good size for allocation size_t mi_good_size(size_t size) mi_attr_noexcept { - if (size <= MI_LARGE_SIZE_MAX) { + if (size <= MI_LARGE_OBJ_SIZE_MAX) { return _mi_bin_size(_mi_bin(size)); } else { @@ -245,7 +245,7 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_expensive(mi_page_queue_contains(queue, page)); - mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); + mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); if (page->prev != NULL) page->prev->next = page->next; if (page->next != NULL) page->next->prev = page->prev; if (page == queue->last) queue->last = page->prev; @@ -268,7 +268,7 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_ mi_assert_internal(page->heap == NULL); mi_assert_internal(!mi_page_queue_contains(queue, page)); mi_assert_internal(page->block_size == queue->block_size || - (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_huge(queue)) || + (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); mi_page_set_in_full(page, mi_page_queue_is_full(queue)); @@ -297,8 +297,8 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro mi_assert_internal((page->block_size == to->block_size && page->block_size == from->block_size) || (page->block_size == to->block_size && mi_page_queue_is_full(from)) || (page->block_size == from->block_size && mi_page_queue_is_full(to)) || - (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_huge(to)) || - (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_full(to))); + (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) || + (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_full(to))); if (page->prev != NULL) page->prev->next = page->next; if (page->next != NULL) page->next->prev = page->prev; diff --git a/src/page.c b/src/page.c index 7ac7535e..627cb728 100644 --- a/src/page.c +++ b/src/page.c @@ -102,7 +102,7 @@ bool _mi_page_is_valid(mi_page_t* page) { mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id); mi_page_queue_t* pq = mi_page_queue_of(page); mi_assert_internal(mi_page_queue_contains(pq, page)); - mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_SIZE_MAX || mi_page_is_in_full(page)); + mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); mi_assert_internal(mi_heap_contains_queue(page->heap,pq)); } return true; @@ -352,8 +352,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_page_set_has_aligned(page, false); // account for huge pages here - if (page->block_size > MI_LARGE_SIZE_MAX) { - if (page->block_size > MI_HUGE_SIZE_MAX) { + if (page->block_size > MI_LARGE_OBJ_SIZE_MAX) { + if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_decrease(&page->heap->tld->stats.giant, page->block_size); } else { @@ -390,7 +390,7 @@ void _mi_page_retire(mi_page_t* page) { // is the only page left with free blocks. It is not clear // how to check this efficiently though... for now we just check // if its neighbours are almost fully used. - if (mi_likely(page->block_size <= MI_MEDIUM_SIZE_MAX)) { + if (mi_likely(page->block_size <= MI_MEDIUM_OBJ_SIZE_MAX)) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { _mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1); return; // dont't retire after all @@ -707,7 +707,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { if (page != NULL) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); - if (page->block_size > MI_HUGE_SIZE_MAX) { + if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_increase(&heap->tld->stats.giant, block_size); _mi_stat_counter_increase(&heap->tld->stats.giant_count, 1); } @@ -740,7 +740,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept // huge allocation? mi_page_t* page; - if (mi_unlikely(size > MI_LARGE_SIZE_MAX)) { + if (mi_unlikely(size > MI_LARGE_OBJ_SIZE_MAX)) { if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) { page = NULL; } diff --git a/src/segment.c b/src/segment.c index 736345bf..78d5d0bd 100644 --- a/src/segment.c +++ b/src/segment.c @@ -24,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file - small pages (64kb), 64 in one segment - medium pages (512kb), 8 in one segment - large pages (4mb), 1 in one segment - - huge blocks > MI_LARGE_SIZE_MAX (512kb) are directly allocated by the OS + - huge blocks > MI_LARGE_OBJ_SIZE_MAX (512kb) are directly allocated by the OS In any case the memory for a segment is virtual and only committed on demand (i.e. we are careful to not touch the memory @@ -715,13 +715,13 @@ static bool mi_is_good_fit(size_t bsize, size_t size) { mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; - if (block_size <= MI_SMALL_SIZE_MAX || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) { + if (block_size <= MI_SMALL_OBJ_SIZE_MAX || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) { page = mi_segment_small_page_alloc(tld,os_tld); } - else if (block_size <= MI_MEDIUM_SIZE_MAX || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { + else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { page = mi_segment_medium_page_alloc(tld, os_tld); } - else if (block_size < MI_LARGE_SIZE_MAX || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t))) { + else if (block_size < MI_LARGE_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t))) { page = mi_segment_large_page_alloc(tld, os_tld); } else {