add documentation for tracking tools; rename with prefix MI_TRACK_tool

CMakeLists.txt

@@ -10,8 +10,8 @@ option(MI_PADDING           "Enable padding to detect heap block overflow (alway
 option(MI_OVERRIDE          "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON)
 option(MI_XMALLOC           "Enable abort() call on memory allocation failure by default" OFF)
 option(MI_SHOW_ERRORS       "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF)
-option(MI_VALGRIND          "Compile with Valgrind support (adds a small overhead)" OFF)
+option(MI_TRACK_VALGRIND    "Compile with Valgrind support (adds a small overhead)" OFF)
-option(MI_ASAN              "Compile with address sanitizer support (adds a small overhead)" OFF)
+option(MI_TRACK_ASAN        "Compile with address sanitizer support (adds a small overhead)" OFF)
 option(MI_USE_CXX           "Use the C++ compiler to compile the library (instead of the C compiler)" OFF)
 option(MI_SEE_ASM           "Generate assembly files" OFF)
 option(MI_OSX_INTERPOSE     "Use interpose to override standard malloc on macOS" ON)
@@ -25,7 +25,7 @@ option(MI_BUILD_TESTS       "Build test executables" ON)
 option(MI_DEBUG_TSAN        "Build with thread sanitizer (needs clang)" OFF)
 option(MI_DEBUG_UBSAN       "Build with undefined-behavior sanitizer (needs clang++)" OFF)
 option(MI_SKIP_COLLECT_ON_EXIT, "Skip collecting memory on program exit" OFF)
-option(MI_NO_PADDING        "Force no use of padding even in DEBUG mode ets." OFF)
+option(MI_NO_PADDING        "Force no use of padding even in DEBUG mode etc." OFF)
 
 # deprecated options
 option(MI_CHECK_FULL        "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF)
@@ -125,42 +125,39 @@ endif()
 
 if(MI_SECURE)
   message(STATUS "Set full secure build (MI_SECURE=ON)")
-  list(APPEND mi_defines MI_SECURE=4)
+  list(APPEND mi_defines MI_SECURE=4)  
-  #if (MI_VALGRIND)
-  #  message(WARNING "Secure mode is a bit weakened when compiling with Valgrind support as buffer overflow detection is no longer byte-precise (if running without valgrind)")
-  #endif()
 endif()
 
-if(MI_VALGRIND)
+if(MI_TRACK_VALGRIND)
   CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH)
   if (NOT MI_HAS_VALGRINDH)
-    set(MI_VALGRIND OFF)
+    set(MI_TRACK_VALGRIND OFF)
     message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first")
-    message(STATUS  "Compile **without** Valgrind support (MI_VALGRIND=OFF)")
+    message(STATUS  "Compile **without** Valgrind support (MI_TRACK_VALGRIND=OFF)")
   else()
-    message(STATUS "Compile with Valgrind support (MI_VALGRIND=ON)")
+    message(STATUS "Compile with Valgrind support (MI_TRACK_VALGRIND=ON)")
-    list(APPEND mi_defines MI_VALGRIND=1)
+    list(APPEND mi_defines MI_TRACK_VALGRIND=1)
   endif()
 endif()
 
-if(MI_ASAN)
+if(MI_TRACK_ASAN)
   if (APPLE AND MI_OVERRIDE)
-    set(MI_ASAN OFF)
+    set(MI_TRACK_ASAN OFF)
-    message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_ASAN=OFF)")
+    message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK_ASAN=OFF)")
   endif()
-  if (MI_VALGRIND)
+  if (MI_TRACK_VALGRIND)
-    set(MI_ASAN OFF)
+    set(MI_TRACK_ASAN OFF)
-    message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_ASAN=OFF)")
+    message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_TRACK_ASAN=OFF)")
   endif()
-  if(MI_ASAN)
+  if(MI_TRACK_ASAN)
     CHECK_INCLUDE_FILES("sanitizer/asan_interface.h" MI_HAS_ASANH)
     if (NOT MI_HAS_ASANH)
-      set(MI_ASAN OFF)
+      set(MI_TRACK_ASAN OFF)
       message(WARNING "Cannot find the 'sanitizer/asan_interface.h' -- install address sanitizer support first")
-      message(STATUS  "Compile **without** address sanitizer support (MI_ASAN=OFF)")
+      message(STATUS  "Compile **without** address sanitizer support (MI_TRACK_ASAN=OFF)")
     else()
-      message(STATUS "Compile with address sanitizer support (MI_ASAN=ON)")
+      message(STATUS "Compile with address sanitizer support (MI_TRACK_ASAN=ON)")
-      list(APPEND mi_defines MI_ASAN=1)
+      list(APPEND mi_defines MI_TRACK_ASAN=1)
       list(APPEND mi_cflags -fsanitize=address)
       list(APPEND mi_libraries -fsanitize=address)
     endif()
@@ -327,10 +324,10 @@ set(mi_basename "mimalloc")
 if(MI_SECURE)
   set(mi_basename "${mi_basename}-secure")
 endif()
-if(MI_VALGRIND)
+if(MI_TRACK_VALGRIND)
   set(mi_basename "${mi_basename}-valgrind")
 endif()
-if(MI_ASAN)
+if(MI_TRACK_ASAN)
   set(mi_basename "${mi_basename}-asan")
 endif()
 string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC)

include/mimalloc-track.h

@@ -1,5 +1,5 @@
 /* ----------------------------------------------------------------------------
-Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
 This is free software; you can redistribute it and/or modify it under the
 terms of the MIT license. A copy of the license can be found in the file
 "LICENSE" at the root of this distribution.
@@ -9,20 +9,39 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MIMALLOC_TRACK_H
 
 /* ------------------------------------------------------------------------------------------------------
-Track memory ranges with macros for tools like Valgrind
+Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.
-address sanitizer, or other memory checkers.
+These can be defined for tracking allocation:
+
+  #define mi_track_malloc_size(p,reqsize,size,zero)
+  #define mi_track_free_size(p,_size)
 
 The macros are set up such that the size passed to `mi_track_free_size`
-matches the size of the allocation, or the new size of a `mi_track_resize` (currently unused though).
+always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).
+The `reqsize` is what the user requested, and `size >= reqsize`.
+The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,
+or otherwise it is the usable block size which may be larger than the original request.
+Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).
+The `zero` parameter is `true` if the allocated block is zero initialized.
+
+Optional:
+
+  #define mi_track_align(p,alignedp,offset,size)
+  #define mi_track_resize(p,oldsize,newsize)
+
+The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.
+The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).
+The `mi_track_resize` is currently unused but could be called on reallocations within a block.
+
+The following macros are for tools like asan and valgrind to track whether memory is 
+defined, undefined, or not accessible at all:
+
+  #define mi_track_mem_defined(p,size)
+  #define mi_track_mem_undefined(p,size)
+  #define mi_track_mem_noaccess(p,size)
 
-The `size` is either byte precise (and what the user requested) if `MI_PADDING` is enabled,
-or otherwise it is the full block size which may be larger than the original request.
-Aligned pointers in a block are signaled right after a `mi_track_malloc`
-with the `mi_track_align` macro. The corresponding `mi_track_free` still 
-uses the block start pointer and original size (corresponding to the `mi_track_malloc`).
 -------------------------------------------------------------------------------------------------------*/
 
-#if MI_VALGRIND
+#if MI_TRACK_VALGRIND
 
 #define MI_TRACK_ENABLED      1
 #define MI_TRACK_HEAP_DESTROY 1           // track free of individual blocks on heap_destroy
@@ -38,7 +57,7 @@ uses the block start pointer and original size (corresponding to the `mi_track_m
 #define mi_track_mem_undefined(p,size)            VALGRIND_MAKE_MEM_UNDEFINED(p,size)
 #define mi_track_mem_noaccess(p,size)             VALGRIND_MAKE_MEM_NOACCESS(p,size)
 
-#elif MI_ASAN
+#elif MI_TRACK_ASAN
 
 #define MI_TRACK_ENABLED      1
 #define MI_TRACK_HEAP_DESTROY 0
@@ -68,6 +87,9 @@ uses the block start pointer and original size (corresponding to the `mi_track_m
 
 #endif
 
+// -------------------
+// Utility definitions
+
 #ifndef mi_track_resize
 #define mi_track_resize(p,oldsize,newsize)      mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false)
 #endif

include/mimalloc-types.h

@@ -29,8 +29,9 @@ terms of the MIT license. A copy of the license can be found in the file
 // Define NDEBUG in the release version to disable assertions.
 // #define NDEBUG
 
-// Define MI_VALGRIND to enable valgrind support
+// Define MI_TRACK_<tool> to enable tracking support
-// #define MI_VALGRIND 1
+// #define MI_TRACK_VALGRIND 1
+// #define MI_TRACK_ASAN     1
 
 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
 // #define MI_STAT 1
@@ -59,7 +60,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 // Reserve extra padding at the end of each block to be more resilient against heap block overflows.
 // The padding can detect buffer overflow on free.
-#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_VALGRIND || MI_ASAN)
+#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_TRACK_VALGRIND || MI_TRACK_ASAN)
 #define MI_PADDING  1
 #endif
 

include/mimalloc.h

@@ -1,5 +1,5 @@
 /* ----------------------------------------------------------------------------
-Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
 This is free software; you can redistribute it and/or modify it under the
 terms of the MIT license. A copy of the license can be found in the file
 "LICENSE" at the root of this distribution.

readme.md

@@ -78,7 +78,7 @@ Note: the `v2.x` version has a new algorithm for managing internal mimalloc page
   and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance
   (see [below](#performance)); please report if you observe any significant performance regression.
 
-* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with asan and improved [Valgrind] support. Support abitrary large
+* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with [#asan] and improved [#Valgrind] support. Support abitrary large
   alignments (in particular for `std::pmr` pools). 
   Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). 
   Heap walks now visit all object (including huge objects). Support Windows nano server containers (by Johannes Schindelin,@dscho). 
@@ -347,16 +347,19 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to
 - Double free's, and freeing invalid heap pointers are detected.
 - Corrupted free-lists and some forms of use-after-free are detected.
 
-## Valgrind
+## Tools
 
-Generally, we recommend using the standard allocator with the amazing [Valgrind] tool (and
+Generally, we recommend using the standard allocator with memory tracking tools, but mimalloc
-also for other address sanitizers).
+can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. 
-However, it is possible to build mimalloc with Valgrind support. This has a small performance
+This has a small performance overhead but does allow detecting memory leaks and byte-precise 
-overhead but does allow detecting memory leaks and byte-precise buffer overflows directly on final
+buffer overflows directly on final executables. See also the `test/test-wrong.c` file to test with various tools.
-executables. To build with valgrind support, use the `MI_VALGRIND=ON` cmake option:
+
+### Valgrind
+
+To build with valgrind support, use the `MI_TRACK_VALGRIND=ON` cmake option:
 
 ```
-> cmake ../.. -DMI_VALGRIND=ON
+> cmake ../.. -DMI_TRACK_VALGRIND=ON
 ```
 
 This can also be combined with secure mode or debug mode.
@@ -385,6 +388,35 @@ Valgrind support is in its initial development -- please report any issues.
 [Valgrind]: https://valgrind.org/
 [valgrind-soname]: https://valgrind.org/docs/manual/manual-core.html#opt.soname-synonyms
 
+### ASAN
+
+To build with the address sanitizer, use the `-DMI_TRACK_ASAN=ON` cmake option:
+
+```
+> cmake ../.. -DMI_TRACK_ASAN=ON
+```
+
+This can also be combined with secure mode or debug mode. 
+You can then run your programs as:'
+
+```
+> ASAN_OPTIONS=verbosity=1 <myprogram>
+```
+
+When you link a program with an address sanitizer build of mimalloc, you should
+generally compile that program too with the address sanitizer enabled. 
+For example, assuming you build mimalloc in `out/debug`:
+
+```
+clang -g -o test-wrong -Iinclude test/test-wrong.c out/debug/libmimalloc-asan-debug.a -lpthread -fsanitize=address -fsanitize-recover=address
+```
+
+Since the address sanitizer redirects the standard allocation functions, on some platforms (macOSX for example)
+it is required to compile mimalloc with `-DMI_OVERRIDE=OFF`.
+Adress sanitizer support is in its initial development -- please report any issues.
+
+[asan]: https://github.com/google/sanitizers/wiki/AddressSanitizer
+
 
 # Overriding Standard Malloc
 

test/test-api-fill.c

@@ -309,7 +309,7 @@ bool check_zero_init(uint8_t* p, size_t size) {
 
 #if MI_DEBUG >= 2
 bool check_debug_fill_uninit(uint8_t* p, size_t size) {
-#if MI_VALGRIND
+#if MI_TRACK_VALGRIND
   (void)p; (void)size;
   return true; // when compiled with valgrind we don't init on purpose
 #else
@@ -325,7 +325,7 @@ bool check_debug_fill_uninit(uint8_t* p, size_t size) {
 }
 
 bool check_debug_fill_freed(uint8_t* p, size_t size) {
-#if MI_VALGRIND
+#if MI_TRACK_VALGRIND
   (void)p; (void)size;
   return true; // when compiled with valgrind we don't fill on purpose
 #else

test/test-wrong.c

@@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file
    Compile in an "out/debug" folder:
 
    > cd out/debug
-   > cmake ../.. -DMI_VALGRIND=1
+   > cmake ../.. -DMI_TRACK_VALGRIND=1
    > make -j8
 
    and then compile this file as:
@@ -29,7 +29,7 @@ terms of the MIT license. A copy of the license can be found in the file
    Compile in an "out/debug" folder:
 
    > cd out/debug
-   > cmake ../.. -DMI_ASAN=1
+   > cmake ../.. -DMI_TRACK_ASAN=1
    > make -j8
 
    and then compile this file as: