diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index b320f690..2140cadd 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -300,7 +300,7 @@ static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { } // Is memory zero initialized? -static inline bool mi_mem_is_zero(void* p, size_t size) { +static inline bool mi_mem_is_zero(const void* p, size_t size) { for (size_t i = 0; i < size; i++) { if (((uint8_t*)p)[i] != 0) return false; } diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 60408ed3..9851ef5e 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -285,7 +285,7 @@ typedef struct mi_page_s { uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` uint8_t segment_in_use:1; // `true` if the segment allocated this page uint8_t is_committed:1; // `true` if the page virtual memory is committed - uint8_t is_zero_init:1; // `true` if the page was zero initialized + uint8_t is_zero_init:1; // `true` if the page was initially zero initialized // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` diff --git a/src/alloc.c b/src/alloc.c index b17adf45..3edc0b51 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -46,12 +46,17 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz // zero the block? note: we need to zero the full block size (issue #63) if mi_unlikely(zero) { mi_assert_internal(page->xblock_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic) - const size_t zsize = (page->is_zero ? sizeof(block->next) + MI_PADDING_SIZE : page->xblock_size); - _mi_memzero_aligned(block, zsize - MI_PADDING_SIZE); + if (page->is_zero) { + block->next = 0; + } + else { + mi_assert_internal(page->xblock_size >= MI_PADDING_SIZE); + _mi_memzero_aligned(block, page->xblock_size - MI_PADDING_SIZE); + } } #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - if (!page->is_zero && !zero && !mi_page_is_huge(page)) { + if (!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); } #elif (MI_SECURE!=0) diff --git a/src/page.c b/src/page.c index 3445e504..27c6fd09 100644 --- a/src/page.c +++ b/src/page.c @@ -639,11 +639,6 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) // enable the new free list page->capacity += (uint16_t)extend; mi_stat_increase(tld->stats.page_committed, extend * bsize); - - // extension into zero initialized memory preserves the zero'd free list - if (!page->is_zero_init) { - page->is_zero = false; - } mi_assert_expensive(mi_page_is_valid_init(page)); } @@ -667,12 +662,12 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi page->keys[0] = _mi_heap_random_next(heap); page->keys[1] = _mi_heap_random_next(heap); #endif - #if MI_DEBUG > 0 - page->is_zero = false; // ensure in debug mode we initialize with MI_DEBUG_UNINIT, see issue #501 - #else page->is_zero = page->is_zero_init; + #if MI_DEBUG>1 + if (page->is_zero_init) { + mi_mem_is_zero(page_start, page_size); + } #endif - mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); mi_assert_internal(page->used == 0); diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index de50f594..4349f578 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -441,7 +441,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) { MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node); - *is_zero = true; + *is_zero = false; *addr = NULL; return ENOMEM; } diff --git a/test/test-stress.c b/test/test-stress.c index 0d6d0bc9..0dccec0a 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -20,6 +20,7 @@ terms of the MIT license. #include #include #include +#include // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // @@ -106,6 +107,7 @@ static void* alloc_items(size_t items, random_t r) { uintptr_t* p = (uintptr_t*)custom_calloc(items,sizeof(uintptr_t)); if (p != NULL) { for (uintptr_t i = 0; i < items; i++) { + assert(p[i] == 0); p[i] = (items - i) ^ cookie; } }