From d70353c0d87d9cb6c6ed8d7ab76c54dd93edd43a Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Fri, 4 Sep 2026 12:58:26 +0200 Subject: [PATCH 1/5] Paste-in mimalloc sources from microsoft/mimalloc@477c1d7 --- system/lib/mimalloc/src/alloc-override.c | 19 ++----------------- .../lib/mimalloc/src/prim/emscripten/prim.c | 12 ++++++++---- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/system/lib/mimalloc/src/alloc-override.c b/system/lib/mimalloc/src/alloc-override.c index 702f92c4486aa..93d066dc1403a 100644 --- a/system/lib/mimalloc/src/alloc-override.c +++ b/system/lib/mimalloc/src/alloc-override.c @@ -32,8 +32,7 @@ typedef void* mi_nothrow_t; #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); #else - // XXX EMSCRIPTEN: Add "weak" - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), weak)); + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); #endif #define MI_FORWARD1(fun,x) MI_FORWARD(fun) #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) @@ -329,18 +328,16 @@ extern "C" { // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) - #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) && !defined(__EMSCRIPTEN__) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) #endif // No forwarding here due to aliasing/name mangling issues - mi_decl_weak // XXX EMSCRIPTEN void* valloc(size_t size) { return mi_valloc(size); } void vfree(void* p) { mi_free(p); } size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } - mi_decl_weak // XXX EMSCRIPTEN int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } // `aligned_alloc` is only available when __USE_ISOC11 is defined. @@ -351,7 +348,6 @@ extern "C" { // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. #if !defined(__GLIBC__) || __USE_ISOC11 - mi_decl_weak // XXX EMSCRIPTEN void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #endif #endif @@ -359,12 +355,10 @@ extern "C" { // no forwarding here due to aliasing/name mangling issues void cfree(void* p) { mi_free(p); } void* pvalloc(size_t size) { return mi_pvalloc(size); } -mi_decl_weak // XXX EMSCRIPTEN void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } #if !defined(_WIN32) void* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); } #endif -mi_decl_weak // XXX EMSCRIPTEN void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } // some systems define reallocarr so mark it as a weak symbol (#751) mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } @@ -375,17 +369,8 @@ mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_r void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size) void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) void __libc_free(void* p) MI_FORWARD0(mi_free, p) - mi_decl_weak // XXX EMSCRIPTEN void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -#ifdef __EMSCRIPTEN__ // emscripten adds some more on top of WASI - void* emscripten_builtin_malloc(size_t size) MI_FORWARD1(mi_malloc, size) - void* emscripten_builtin_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) - void emscripten_builtin_free(void* p) MI_FORWARD0(mi_free, p) - void* emscripten_builtin_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } - void* emscripten_builtin_calloc(size_t nmemb, size_t size) MI_FORWARD2(mi_calloc, nmemb, size) -#endif - #elif defined(__linux__) // forward __libc interface (needed for glibc-based and musl-based Linux distributions) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) diff --git a/system/lib/mimalloc/src/prim/emscripten/prim.c b/system/lib/mimalloc/src/prim/emscripten/prim.c index d0290778da82d..b52735ae270ff 100644 --- a/system/lib/mimalloc/src/prim/emscripten/prim.c +++ b/system/lib/mimalloc/src/prim/emscripten/prim.c @@ -75,7 +75,7 @@ extern void* emmalloc_memalign(size_t alignment, size_t size); // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { - MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); + MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; // TODO: Track the highest address ever seen; first uses of it are zeroes. // That assumes no one else uses sbrk but us (they could go up, @@ -86,8 +86,12 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm if (try_alignment < MIN_EMMALLOC_ALIGN) { try_alignment = MIN_EMMALLOC_ALIGN; } - *addr = emmalloc_memalign(try_alignment, size); - return (*addr != NULL ? 0 : ENOMEM); + void* p = emmalloc_memalign(try_alignment, size); + *addr = p; + if (p == 0) { + return ENOMEM; + } + return 0; } @@ -173,7 +177,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) #include void _mi_prim_out_stderr( const char* msg) { - emscripten_err(msg); + emscripten_console_error(msg); } From 1836e658bb3f2aa43dda51acc7bd984328916384 Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Fri, 4 Sep 2026 13:05:01 +0200 Subject: [PATCH 2/5] Update mimalloc to 3.5.1 (microsoft/mimalloc@34fbd7e) --- system/lib/mimalloc/README.emscripten | 2 +- .../mimalloc/include/mimalloc-new-delete.h | 2 +- system/lib/mimalloc/include/mimalloc-stats.h | 7 +- system/lib/mimalloc/include/mimalloc.h | 51 +- system/lib/mimalloc/include/mimalloc/atomic.h | 39 +- system/lib/mimalloc/include/mimalloc/bits.h | 19 +- .../lib/mimalloc/include/mimalloc/internal.h | 550 +++++--- .../lib/mimalloc/include/mimalloc/prim-tls.h | 424 ++++++ system/lib/mimalloc/include/mimalloc/prim.h | 436 +------ system/lib/mimalloc/include/mimalloc/track.h | 8 +- system/lib/mimalloc/include/mimalloc/types.h | 176 ++- system/lib/mimalloc/readme.md | 184 ++- system/lib/mimalloc/src/alloc-aligned.c | 115 +- system/lib/mimalloc/src/alloc-override.c | 19 +- system/lib/mimalloc/src/alloc-posix.c | 24 +- system/lib/mimalloc/src/alloc.c | 363 ++++-- system/lib/mimalloc/src/arena-meta.c | 179 --- system/lib/mimalloc/src/arena.c | 566 +++++--- system/lib/mimalloc/src/bitmap.c | 304 +++-- system/lib/mimalloc/src/bitmap.h | 17 +- system/lib/mimalloc/src/free.c | 461 ++++--- system/lib/mimalloc/src/heap.c | 239 ++-- system/lib/mimalloc/src/init.c | 1136 +++++------------ system/lib/mimalloc/src/libc.c | 139 +- system/lib/mimalloc/src/options.c | 114 +- system/lib/mimalloc/src/os.c | 228 ++-- system/lib/mimalloc/src/page-map.c | 375 +++--- system/lib/mimalloc/src/page-queue.c | 39 +- system/lib/mimalloc/src/page.c | 367 +++--- .../lib/mimalloc/src/prim/emscripten/prim.c | 22 +- .../src/prim/osx/alloc-override-zone.c | 15 +- system/lib/mimalloc/src/prim/prim-tls.c | 252 ++++ system/lib/mimalloc/src/prim/prim.c | 2 +- system/lib/mimalloc/src/prim/readme.md | 3 + system/lib/mimalloc/src/prim/unix/prim.c | 178 +-- system/lib/mimalloc/src/prim/windows/prim.c | 218 +++- system/lib/mimalloc/src/random.c | 6 +- system/lib/mimalloc/src/static.c | 3 +- system/lib/mimalloc/src/stats.c | 87 +- system/lib/mimalloc/src/subproc.c | 327 +++++ system/lib/mimalloc/src/theap.c | 413 +++--- system/lib/mimalloc/src/threadlocal.c | 140 +- tools/system_libs.py | 1 + 43 files changed, 4738 insertions(+), 3512 deletions(-) create mode 100644 system/lib/mimalloc/include/mimalloc/prim-tls.h delete mode 100644 system/lib/mimalloc/src/arena-meta.c create mode 100644 system/lib/mimalloc/src/prim/prim-tls.c create mode 100644 system/lib/mimalloc/src/subproc.c diff --git a/system/lib/mimalloc/README.emscripten b/system/lib/mimalloc/README.emscripten index 57b1d6c1b1c06..fa67c1315811f 100644 --- a/system/lib/mimalloc/README.emscripten +++ b/system/lib/mimalloc/README.emscripten @@ -1,5 +1,5 @@ -This contains mimalloc 477c1d71c75a92255b3bdffb6f9dcae0c31d21c8 (v3.4.1) with +This contains mimalloc 34fbd7e7cd4627424490afe19b20f8066bfc537d (v3.5.1) with Emscripten-specific changes. Origin: https://github.com/microsoft/mimalloc diff --git a/system/lib/mimalloc/include/mimalloc-new-delete.h b/system/lib/mimalloc/include/mimalloc-new-delete.h index c16f4a6653d9c..aaf185bb1499c 100644 --- a/system/lib/mimalloc/include/mimalloc-new-delete.h +++ b/system/lib/mimalloc/include/mimalloc-new-delete.h @@ -48,7 +48,7 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); }; #endif - #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) + #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; diff --git a/system/lib/mimalloc/include/mimalloc-stats.h b/system/lib/mimalloc/include/mimalloc-stats.h index 1c620fd1c1f75..f8be41aa6f8f1 100644 --- a/system/lib/mimalloc/include/mimalloc-stats.h +++ b/system/lib/mimalloc/include/mimalloc-stats.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2024-2025, Microsoft Research, Daan Leijen +Copyright (c) 2024-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file #elif __cplusplus >= 201103L #define mi_decl_align(a) alignas(a) #else -#define mi_decl_align(a) +#define mi_decl_align(a) _Alignas(a) #endif @@ -140,6 +140,9 @@ mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_ mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept; +// stats from a theap +mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; + // stats from a subprocess and its heaps aggregated mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept; mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL diff --git a/system/lib/mimalloc/include/mimalloc.h b/system/lib/mimalloc/include/mimalloc.h index 14c06c07c57c0..d389886132321 100644 --- a/system/lib/mimalloc/include/mimalloc.h +++ b/system/lib/mimalloc/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30401 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30501 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -119,7 +119,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* // ------------------------------------------------------ // Extended allocation functions // ------------------------------------------------------ -#define MI_SMALL_WSIZE_MAX (128) +#define MI_SMALL_WSIZE_MAX 128 #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -133,10 +133,17 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; +// `mi_free_size` can be more efficient (as it calls mi_free_small internally). +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; + // `mi_free_small` is for special applications like language runtimes. -// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free` +// it should only be used to free objects from `mi_*alloc_small` and is potentially a tiny bit faster than `mi_free` mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; +// As `mi_free_small` but `p` should not be a NULL pointer. +mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept; + + // ------------------------------------------------------------------------------------- // Aligned allocation // Note that `alignment` always follows `size` for consistency with unaligned @@ -157,12 +164,13 @@ mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t new // Typed allocation, the type is always the first parameter // ------------------------------------------------------ -#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) -#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) +#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp))) +#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp))) #define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) #define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) #define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) #define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) +#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp))) #define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) #define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) @@ -187,7 +195,7 @@ typedef void (mi_cdecl mi_error_fun)(int err, void* arg); mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); mi_decl_export void mi_collect(bool force) mi_attr_noexcept; -mi_decl_export int mi_version(void) mi_attr_noexcept; +mi_decl_export int mi_version(void); mi_decl_export void mi_options_print(void) mi_attr_noexcept; mi_decl_export void mi_process_info_print(void) mi_attr_noexcept; mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; @@ -329,6 +337,7 @@ mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; mi_decl_export void mi_arenas_print(void) mi_attr_noexcept; mi_decl_export size_t mi_arena_min_alignment(void); mi_decl_export size_t mi_arena_min_size(void); +mi_decl_export size_t mi_arena_max_object_size(void); typedef void* mi_arena_id_t; mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); @@ -382,8 +391,33 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); + +// ------------------------------------------------------ +// Fast constant size allocations. +// ------------------------------------------------------ +static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_malloc_small(size); } else { return mi_malloc(size); } +} +static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_zalloc_small(size); } else { return mi_zalloc(size); } +} +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +} +static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_zalloc(theap,size); } +} +static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } +} +static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { + // assert(p!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } +} // ------------------------------------------------------ // Experimental @@ -465,7 +499,7 @@ typedef enum mi_option_e { mi_option_deprecated_purge_extend_delay, mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) - mi_option_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0) + mi_option_deprecated_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0) mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0) mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) @@ -513,7 +547,7 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing. // ------------------------------------------------------------------------------------------------------- -mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; @@ -530,7 +564,6 @@ mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, s mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; -mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; diff --git a/system/lib/mimalloc/include/mimalloc/atomic.h b/system/lib/mimalloc/include/mimalloc/atomic.h index e8960293cc4ea..9c23398024ab7 100644 --- a/system/lib/mimalloc/include/mimalloc/atomic.h +++ b/system/lib/mimalloc/include/mimalloc/atomic.h @@ -9,13 +9,14 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ATOMIC_H // include windows.h or pthreads.h -#if defined(_WIN32) +#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it at unix #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include -#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)) -#define MI_USE_PTHREADS +#endif +#if MI_TLS_MODEL_PTHREADS || (!defined(_WIN32) && !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) +#define MI_USE_PTHREADS 1 #include #endif @@ -215,24 +216,22 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt (void)(mo); return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); } -static inline void mi_atomic_thread_fence(mi_memory_order mo) { - (void)(mo); - _Atomic(uintptr_t) x = 0; - mi_atomic_exchange_explicit(&x, 1, mo); -} static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { (void)(mo); // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc #if defined(_M_IX86) || defined(_M_X64) + // on x86/x64 we have a strong memory model so any load is acquire return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); #elif defined(_M_ARM) || defined(_M_ARM64) if (mo == mi_memory_order_relaxed) { return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); } + #if !defined(__clang__) // work around __ldar missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_acquire) { return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p); } + #endif else { const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); __dmb(15); // _ARM(64)_BARRIER_SY @@ -252,9 +251,11 @@ static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, m if (mo == mi_memory_order_relaxed) { MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x); } + #if !defined(__clang__) // work around __stlr missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_release) { MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x); } + #endif else { mi_atomic_exchange_explicit(p, x, mo); } @@ -272,7 +273,7 @@ static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_o if (mo == mi_memory_order_relaxed) { return __iso_volatile_load64((volatile const int64_t*)p); } - #if defined(_M_ARM64) + #if defined(_M_ARM64) && !defined(__clang__) // work around __ldar64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_acquire) { return __ldar64((volatile const uintptr_t*)p); } @@ -297,7 +298,7 @@ static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi if (mo == mi_memory_order_relaxed) { __iso_volatile_store64((volatile int64_t*)p,x); } - #if defined(_M_ARM64) + #if defined(_M_ARM64) && !defined(__clang__) // work around __stlr64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo == mi_memory_order_release) { __stlr64((volatile uint64_t*)p, (uint64_t)x); } @@ -335,18 +336,18 @@ static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const vola } } -static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) { +static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) { int64_t current; do { current = *p; } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); } -static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) { +static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t)* p, int64_t i) { mi_atomic_addi64_relaxed(p, i); } -static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) { +static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, int64_t* exp, int64_t des) { const int64_t read = _InterlockedCompareExchange64(p, des, *exp); if (read == *exp) { return true; @@ -358,17 +359,17 @@ static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, } // The pointer macros cast to `uintptr_t`. -#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)) -#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)) +#define mi_atomic_load_ptr_acquire(tp,p) ((tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))) +#define mi_atomic_load_ptr_relaxed(tp,p) ((tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))) #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) #define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) -#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x) -#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x) -#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) ((tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) +#define mi_atomic_exchange_ptr_release(tp,p,x) ((tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) ((tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) #define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) #define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) @@ -417,7 +418,7 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t; #define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) ) -#if defined(_WIN32) +#if defined(_WIN32) || defined(__CYGWIN__) typedef struct mi_lock_s { SRWLOCK mutex; // slim reader-writer lock diff --git a/system/lib/mimalloc/include/mimalloc/bits.h b/system/lib/mimalloc/include/mimalloc/bits.h index 074d7f594f629..c7ce294f9870f 100644 --- a/system/lib/mimalloc/include/mimalloc/bits.h +++ b/system/lib/mimalloc/include/mimalloc/bits.h @@ -103,13 +103,13 @@ typedef int32_t mi_ssize_t; #include #endif -#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc +#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // avx2 implies bmi2 #define __BMI2__ 1 #endif -#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc +#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__) || defined(__BMI__)) && !defined(__BMI1__) // bmi2 implies bmi1 #define __BMI1__ 1 #endif -#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc +#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // avx2 implies lzcnt #define __LZCNT__ 1 #endif @@ -127,9 +127,18 @@ typedef int32_t mi_ssize_t; #define MI_MAX_VABITS (32) #endif +// the MI_MIN_VABITS determine how many bits of the address space are always committed in the page_map +#if MI_MAX_VABITS <= 32 +#define MI_MIN_VABITS (32) +#elif MI_MAX_VABITS <= 42 +#define MI_MIN_VABITS MI_MAX_VABITS +#else +#define MI_MIN_VABITS (43) /* 8 TiB */ +#endif + // use a flat page-map or a 2-level one #ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP #define MI_PAGE_MAP_FLAT 1 #else #define MI_PAGE_MAP_FLAT 0 @@ -184,7 +193,7 @@ static inline size_t mi_popcount(size_t x) { #if mi_has_builtinz(popcount) return mi_builtinz(popcount)(x); #elif defined(_MSC_VER) && (MI_ARCH_ARM64 || MI_ARCH_ARM32) - return mi_msc_builtinz(__popcnt)(x); + return mi_msc_builtinz(_CountOneBits)(x); #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86) if (_mi_cpu_has_popcnt) { return mi_msc_builtinz(__popcnt)(x); } else { return _mi_popcount_generic(x); } // see issue #1291 diff --git a/system/lib/mimalloc/include/mimalloc/internal.h b/system/lib/mimalloc/include/mimalloc/internal.h index 7deaceff29f44..fdd5c78bed9dd 100644 --- a/system/lib/mimalloc/include/mimalloc/internal.h +++ b/system/lib/mimalloc/include/mimalloc/internal.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -50,7 +50,11 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_thread __thread #define mi_decl_noreturn __attribute__((noreturn)) #define mi_decl_weak __attribute__((weak)) +#if defined(__MINGW32__) || defined(__CYGWIN__) +#define mi_decl_hidden +#else #define mi_decl_hidden __attribute__((visibility("hidden"))) +#endif #if (__GNUC__ >= 4) || defined(__clang__) #define mi_decl_cold __attribute__((cold)) #else @@ -85,16 +89,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_likely(x) (x) #endif -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#else -#define mi_decl_externc -#endif - #if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc #define mi_decl_maybe_unused __attribute__((unused)) #elif __cplusplus >= 201703L // c++17 @@ -103,10 +97,16 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_maybe_unused #endif +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + #if defined(__cplusplus) #define mi_decl_externc extern "C" +#define mi_init_struct_zero { } #else #define mi_decl_externc +#define mi_init_struct_zero { 0 } #endif @@ -126,19 +126,19 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); char _mi_toupper(char c); int _mi_strnicmp(const char* s, const char* t, size_t n); -void _mi_strlcpy(char* dest, const char* src, size_t dest_size); -void _mi_strlcat(char* dest, const char* src, size_t dest_size); +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied +bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appended size_t _mi_strlen(const char* s); size_t _mi_strnlen(const char* s, size_t max_len); char* _mi_strnstr(char* s, size_t max_len, const char* pat); bool _mi_streq(const char* s, const char* t); int _mi_getenv(const char* name, char* result, size_t result_size); +void _mi_detect_cpu_features(void); // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); void _mi_raw_message(const char* fmt, ...); -void _mi_message(const char* fmt, ...); void _mi_warning_message(const char* fmt, ...); void _mi_verbose_message(const char* fmt, ...); void _mi_trace_message(const char* fmt, ...); @@ -157,74 +157,88 @@ uintptr_t _mi_theap_random_next(mi_theap_t* theap); uintptr_t _mi_os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); +// prim-tls.c +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + +// subproc.c +mi_subproc_t* _mi_subproc_main_init(void); +void _mi_subproc_main_done(void); +mi_subproc_t* _mi_subproc_main(void); +bool _mi_subproc_is_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc(void); // current subproc of this thread +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_subprocs_unsafe_destroy_all(void); + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid ); +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid ); +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid); +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p); + + // init.c -extern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty; +mi_page_t* _mi_page_empty_get(void); void _mi_auto_process_init(void); void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; bool _mi_is_redirected(void); bool _mi_allocator_init(const char** message); void _mi_allocator_done(void); -bool _mi_is_main_thread(void); bool _mi_preloading(void); // true while the C runtime is not initialized yet void _mi_thread_done(mi_theap_t* theap); - -mi_subproc_t* _mi_subproc(void); -mi_subproc_t* _mi_subproc_main(void); -mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); -mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); - -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; -size_t _mi_thread_seq_id(void) mi_attr_noexcept; -bool _mi_is_heap_main(const mi_heap_t* heap); -bool _mi_is_theap_main(const mi_theap_t* theap); -void _mi_theap_guarded_init(mi_theap_t* theap); -void _mi_theap_options_init(mi_theap_t* theap); -mi_theap_t* _mi_theap_default_safe(void); // ensure the returned theap is initialized -mi_theap_t* _mi_theap_main_safe(void); +mi_theap_t* _mi_thread_init(void); +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); +bool _mi_is_empty_theap(const mi_theap_t* theap); // os.c void _mi_os_init(void); // called from process init -void* _mi_os_alloc(size_t size, mi_memid_t* memid); -void* _mi_os_zalloc(size_t size, mi_memid_t* memid); -void _mi_os_free(void* p, size_t size, mi_memid_t memid); -void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc ); +void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); +void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); +void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); +void _mi_os_free_ex(mi_subproc_t* subproc, void* p, size_t size, bool still_committed, mi_memid_t memid ); size_t _mi_os_page_size(void); size_t _mi_os_guard_page_size(void); size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); bool _mi_os_has_virtual_reserve(void); +bool _mi_os_canuse_thp(void); size_t _mi_os_virtual_address_bits(void); size_t _mi_os_minimal_purge_size(void); -bool _mi_os_reset(void* addr, size_t size); -bool _mi_os_decommit(void* addr, size_t size); -void _mi_os_reuse(void* p, size_t size); -mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero); -mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size); +bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size); +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); +void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); +mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); +mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); -bool _mi_os_purge(void* p, size_t size); -bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg); +bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); +bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg); size_t _mi_os_secure_guard_page_size(void); -bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); int _mi_os_numa_node(void); int _mi_os_numa_node_count(void); -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); -void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); +void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); +void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); bool _mi_os_canuse_large_page(size_t size, size_t alignment); size_t _mi_os_large_page_size(void); -void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); +void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); // threadlocal.c +#define mi_thread_local_key_fast ((mi_thread_local_t)1) mi_thread_local_t _mi_thread_local_create(void); void _mi_thread_local_free( mi_thread_local_t key ); @@ -241,8 +255,7 @@ bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_ void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); -void _mi_arenas_free(void* p, size_t size, mi_memid_t memid); -bool _mi_arenas_contain(const void* p); +void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); @@ -252,30 +265,22 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); -// arena-meta.c -void* _mi_meta_zalloc( size_t size, mi_memid_t* memid ); -void _mi_meta_free(void* p, size_t size, mi_memid_t memid); -bool _mi_meta_is_meta_page(void* p); - // "page-map.c" bool _mi_page_map_init(void); mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page); void _mi_page_map_unregister(mi_page_t* page); void _mi_page_map_unregister_range(void* start, size_t size); mi_page_t* _mi_safe_ptr_page(const void* p); -void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc); +void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... - -size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append); void _mi_deferred_free(mi_theap_t* theap, bool force); - void _mi_page_free_collect(mi_page_t* page, bool force); void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); @@ -286,26 +291,29 @@ size_t _mi_bin_size(size_t bin); // for stats size_t _mi_bin(size_t size); // for stats // "theap.c" +void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld); mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld); -void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock); -void _mi_theap_default_set(mi_theap_t* theap); -void _mi_theap_cached_set(mi_theap_t* theap); void _mi_theap_collect_retired(mi_theap_t* theap, bool force); void _mi_theap_collect_abandon(mi_theap_t* theap); bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page); -bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock); + +void _mi_heap_detach_theaps( mi_heap_t* heap ); +void _mi_tld_detach_theaps( mi_tld_t* tld ); void _mi_theap_incref(mi_theap_t* theap); void _mi_theap_decref(mi_theap_t* theap); -bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ); +void _mi_theap_merge_stats(mi_theap_t* theap); // "heap.c" +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id); void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap -mi_decl_cold mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap); // get the theap for a heap without initializing (and return NULL in that case) void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in "arena.c" void _mi_heap_destroy_pages(mi_heap_t* heap_from); // in "arena.c" -void _mi_heap_force_destroy(mi_heap_t* heap); // allow destroying the main heap +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap +mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main); +bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap); // "stats.c" void _mi_stats_init(void); @@ -317,14 +325,16 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept; -void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept; +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); // "free.c" +void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); @@ -388,16 +398,58 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); #define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount) #define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) -#define mi_os_stat_counter_increase(stat,amount) mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount) -#define mi_os_stat_increase(stat,amount) mi_subproc_stat_increase(_mi_subproc(),stat,amount) -#define mi_os_stat_decrease(stat,amount) mi_subproc_stat_decrease(_mi_subproc(),stat,amount) - #define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) #define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) #define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) #define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) +#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } +#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } + +/* ----------------------------------------------------------- + pthread thread locals +----------------------------------------------------------- */ + +#if MI_USE_PTHREADS + +#if defined(__APPLE__) && defined(__aarch64__) +#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(0)) // nicer codegen +#else +#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(-1)) +#endif + +#if defined(__linux__) && defined(__GLIBC__) +// pthread_getspecific returns NULL for invalid keys. +// see also: +#define MI_PTHREADS_GET_INVALID_KEY_IS_NULL 1 +#endif + +mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init); + +static inline void* mi_pthread_key_get(pthread_key_t key) { + #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL + if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL; + #endif + return pthread_getspecific(key); +} + +static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) { + if mi_likely(*pkey!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(*pkey,val); return true; } + else if (val!=NULL) { return _mi_pthread_key_create(pkey,NULL,val); } + else return true; +} + +static inline void mi_pthread_key_delete(pthread_key_t* pkey) { + const pthread_key_t key = *pkey; + if (key!=MI_PTHREAD_KEY_INVALID) { + *pkey = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); + } +} +#endif /* ----------------------------------------------------------- Options (exposed for the debugger) @@ -449,6 +501,11 @@ static inline bool _mi_is_power_of_two(uintptr_t x) { return ((x & (x - 1)) == 0); } +// valid alignment values are as posix memalign: +static inline bool mi_alignment_is_valid(size_t alignment) { + return ((alignment!=0) && _mi_is_power_of_two(alignment)); +} + // Is a pointer aligned? static inline bool _mi_is_aligned(const void* p, size_t alignment) { return (alignment==0 || ((uintptr_t)p % alignment) == 0); @@ -457,7 +514,7 @@ static inline bool _mi_is_aligned(const void* p, size_t alignment) { // Align upwards static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); - uintptr_t mask = alignment - 1; + const uintptr_t mask = alignment - 1; if ((alignment & mask) == 0) { // power of two? return ((sz + mask) & ~mask); } @@ -471,18 +528,19 @@ static inline void* _mi_align_up_ptr(const void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } +// Align down static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); - uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) { // power of two? + const uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? return (sz & ~mask); } else { - return ((sz / alignment) * alignment); + return ((sz/alignment)*alignment); } } -// align a pointer downwards +// Align a pointer downwards static inline void* _mi_align_down_ptr(const void* p, size_t alignment) { return (void*)_mi_align_down((uintptr_t)p, alignment); } @@ -523,10 +581,10 @@ static inline size_t _mi_wsize_from_size(size_t size) { #undef _CLOCK_T #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #if (SIZE_MAX == ULONG_MAX) - return __builtin_umull_overflow(count, size, (unsigned long *)total); - #elif (SIZE_MAX == UINT_MAX) + #if (SIZE_MAX == UINT_MAX) return __builtin_umul_overflow(count, size, (unsigned int *)total); + #elif (SIZE_MAX == ULONG_MAX) + return __builtin_umull_overflow(count, size, (unsigned long *)total); #else return __builtin_umulll_overflow(count, size, (unsigned long long *)total); #endif @@ -566,16 +624,28 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot Heap functions ------------------------------------------------------------------------------------------- */ -extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL) -extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated +extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) +extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated + +static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) { + return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap); +} static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) { - return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap); + mi_heap_t* const heap = _mi_theap_heap_peek(theap); + mi_assert_internal(heap!=NULL); + return heap; } static inline bool mi_theap_is_initialized(const mi_theap_t* theap) { - return (theap != NULL && _mi_theap_heap(theap) != NULL); + return (theap != NULL && _mi_theap_heap_peek(theap) != NULL); +} + +static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + mi_assert_internal(!mi_theap_is_initialized(theap) || _mi_theap_heap(theap)->subproc == subproc); + return subproc; } static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) { @@ -585,13 +655,14 @@ static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t return theap->pages_free_direct[idx]; } +static inline bool mi_theap_is_detached(mi_theap_t* theap) { + return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED); +} -//static inline uintptr_t _mi_ptr_cookie(const void* p) { -// extern mi_theap_t _mi_theap_main; -// mi_assert_internal(_mi_theap_main.cookie != 0); -// return ((uintptr_t)p ^ _mi_theap_main.cookie); -//} - +static inline bool mi_theap_matches_thread(mi_theap_t* theap) { + const mi_threadid_t tid = _mi_thread_id(); + return (theap==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); +} /* ----------------------------------------------------------- The page map maps addresses to `mi_page_t` pointers @@ -602,20 +673,30 @@ static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t // flat page-map committed on demand, using one byte per slice (64 KiB). // single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses) // used by default on <= 40 bit virtual address spaces. -extern mi_decl_hidden uint8_t* _mi_page_map; +extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map; +extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; static inline size_t _mi_page_map_index(const void* p) { return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT); } +static inline uint8_t _mi_page_map_at(size_t idx) { + return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx]; +} + static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) { const size_t idx = _mi_page_map_index(p); - const size_t ofs = _mi_page_map[idx]; + const size_t ofs = _mi_page_map_at(idx); if (valid != NULL) { *valid = (ofs != 0); } return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT); } static inline mi_page_t* _mi_checked_ptr_page(const void* p) { + #if MI_MIN_VABITS < MI_INTPTR_BITS + if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; + } + #endif bool valid; mi_page_t* const page = _mi_ptr_page_ex(p, &valid); return (valid ? page : NULL); @@ -637,10 +718,17 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { #define MI_PAGE_MAP_SUB_SHIFT (13) #define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) #define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) -#define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT) typedef mi_page_t** mi_submap_t; -extern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map; +typedef struct mi_page_map_s { + _Atomic(size_t) committed_count; // currently committed entries + size_t reserved_size; // full reserved size (mi_page_map_t + submaps) + mi_memid_t memid; // provenance + mi_lock_t lock; // used when allocating new submaps + _Atomic(mi_submap_t) submaps[1]; +} mi_page_map_t; + +extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); @@ -648,33 +736,81 @@ static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { return (u / MI_PAGE_MAP_SUB_COUNT); } -static inline mi_submap_t _mi_page_map_at(size_t idx) { - return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]); +static inline mi_page_map_t* _mi_page_map(void) { + return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map); +} + +static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { + return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); } static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { + const mi_page_map_t* pmap = _mi_page_map(); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - return (_mi_page_map_at(idx))[sub_idx]; // NULL if p==NULL + return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL } static inline mi_page_t* _mi_checked_ptr_page(const void* p) { + const mi_page_map_t* pmap = _mi_page_map(); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - mi_submap_t const sub = _mi_page_map_at(idx); + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + if mi_unlikely(idx >= committed_count) return NULL; + // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized + // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + // if mi_unlikely(idx >= committed_count) return NULL; + // } + // #endif + mi_submap_t const sub = _mi_page_map_at(pmap,idx); if mi_unlikely(sub == NULL) return NULL; return sub[sub_idx]; } #endif +#if MI_PAGE_META_IS_ALIGNED +// if the page meta data is aligned in front of pages we can find it efficiently +// without needing to go through the page map (for valid pointers). +static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); + mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT); + #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 + mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); + #else + mi_page_t* const page = &page_metas[page_idx]; + #endif + return page; + +} + +static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { + mi_page_t* const page = _mi_aligned_ptr_page0(p); + if mi_unlikely(page==NULL) return NULL; + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { + _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p); + return NULL; + } + #endif + return mi_atomic_load_ptr_acquire(mi_page_t, &page->self); +} +#endif static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); - #if MI_DEBUG || MI_SECURE || MI_FREE_IS_CHECKED - return _mi_checked_ptr_page(p); - #else - return _mi_unchecked_ptr_page(p); + #if MI_SECURE || MI_FREE_IS_CHECKED + return _mi_checked_ptr_page(p); + #elif MI_PAGE_META_IS_ALIGNED + return _mi_aligned_ptr_page(p); + #elif MI_DEBUG + return _mi_checked_ptr_page(p); + #else + return _mi_unchecked_ptr_page(p); #endif } @@ -687,8 +823,8 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { - // multiplication must be done in `size_t`; in a 32-bit multiplication the offset wraps for pages whose blocks start 4 GiB or more after the page meta info - return (uint8_t*)page + ((size_t)page->page_woffset * MI_SIZE_SIZE); + // multiplication must be done in `size_t` + return (uint8_t*)page + page->page_offset; } static inline size_t mi_page_size(const mi_page_t* page) { @@ -725,8 +861,16 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) { } static inline bool mi_page_meta_is_separated(const mi_page_t* page) { - #if MI_PAGE_META_IS_SEPARATED - // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + return (page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + #else + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; + #endif + #elif MI_PAGE_META_IS_SEPARATED + // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); #else MI_UNUSED(page); @@ -750,9 +894,15 @@ static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offse return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start; } +// How much of the page is committed relative to the slice start? (or 0 if fully committed already) +static inline size_t mi_page_slice_committed(const mi_page_t* page) { + return ((size_t)page->slice_pcommitted * _mi_os_page_size()); +} + // Currently committed part of a page static inline size_t mi_page_committed(const mi_page_t* page) { - return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0)); + const size_t slice_committed = mi_page_slice_committed(page); + return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); } // are all blocks in a page freed? @@ -810,6 +960,10 @@ static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t siz return pq; } +static inline size_t mi_page_min_commit_size(void) { + const size_t psize = _mi_os_page_size(); + return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize); +} //----------------------------------------------------------- // Page thread id and flags @@ -847,7 +1001,8 @@ static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { mi_theap_t* const theap = page->theap; mi_assert_internal(theap!=NULL); if (theap != NULL) { - const size_t size = page->capacity * mi_page_block_size(page); + mi_assert_internal(page->capacity==page->reserved); + const size_t size = page->reserved * mi_page_block_size(page); if (in_full) { theap->pages_full_size += size; } else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; } } @@ -898,7 +1053,7 @@ static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { static inline mi_theap_t* mi_page_theap(const mi_page_t* page) { mi_assert_internal(!mi_page_is_abandoned(page)); - mi_assert_internal(page->theap != NULL); + mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty); return page->theap; } @@ -911,12 +1066,33 @@ static inline mi_tld_t* mi_page_tld(const mi_page_t* page) { static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { mi_heap_t* heap = page->heap; - // we use NULL for the main heap to make `_mi_page_get_associated_theap` fast in `free.c:mi_abandoned_page_try_reclaim`. - if mi_likely(heap==NULL) heap = mi_heap_main(); mi_assert_internal(heap != NULL); return heap; } +static inline mi_subproc_t* mi_page_subproc(const mi_page_t* page) { + mi_heap_t* const heap = mi_page_heap(page); + return heap->subproc; +} + +static inline mi_heap_t* mi_arena_heap_main(const mi_arena_t* arena) { + return _mi_subproc_heap_main(arena->subproc); +} + +static inline mi_heap_t* mi_heap_get_heap_main(const mi_heap_t* heap) { + return _mi_subproc_heap_main(heap->subproc); +} + +static inline bool _mi_is_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (mi_heap_get_heap_main(heap) == heap); +} + +static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (_mi_subproc_main()->heap_main == heap); +} + //----------------------------------------------------------- // Thread free list and ownership //----------------------------------------------------------- @@ -984,23 +1160,28 @@ static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { theap->guarded_sample_count = count; return false; } - else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { - // use guarded allocation - theap->guarded_sample_count = theap->guarded_sample_rate; // reset - return (theap->guarded_sample_rate != 0); - } else { - // failed size criteria, rewind count (but don't write to an empty theap) - if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; } - return false; + // count == 0 + const size_t rate = theap->guarded_sample_rate; + if (rate == 0) { + return false; // don't write to an empty theap + } + else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_count = rate; // reset + return true; + } + else { + // failed size criteria, rewind count + theap->guarded_sample_count = 1; + return false; + } } } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept; - +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; #endif - /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers @@ -1033,25 +1214,46 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { } static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { - void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); + const uintptr_t k1 = keys[0]; + #if MI_PAGE_KEY_COUNT==2 + const uintptr_t k2 = keys[1]; + #else + const uintptr_t k2 = mi_rotr(k1,13); + #endif + void* p = (void*)(mi_rotr(x - k1, k1) ^ k2); return (p==null ? NULL : p); } static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { - uintptr_t x = (uintptr_t)(p==NULL ? null : p); - return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; + const uintptr_t k1 = keys[0]; + #if MI_PAGE_KEY_COUNT==2 + const uintptr_t k2 = keys[1]; + #else + const uintptr_t k2 = mi_rotr(k1,13); + #endif + const uintptr_t x = (uintptr_t)(p==NULL ? null : p); + return mi_rotl(x ^ k2, k1) + k1; } static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys)); // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951) + // also clear bit 9 which we set only when a block is freed. #if MI_BIG_ENDIAN - return (x & 0x00FFFFFF); + return (x & 0x00FFFEFF); #else - return (x & 0xFFFFFF00); + return (x & 0xFFFFFE00); #endif } +static inline uint32_t mi_ptr_encode_canary_freed(void) { + return (0x00DEAD00); // set bit 9 so it is different from any valid canary +} + +static inline bool mi_ptr_decode_canary_is_freed(uint32_t canary) { + return (canary == mi_ptr_encode_canary_freed()); +} + static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; @@ -1076,14 +1278,15 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_track_mem_noaccess(block,sizeof(mi_block_t)); } +mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c + static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { #if MI_ENCODE_FREELIST mi_block_t* next = mi_block_nextx(page,block,page->keys); // check for free list corruption: is `next` at least in the same page? - // TODO: check if `next` is `page->block_size` aligned? - if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { - _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); - next = NULL; + // todo: check if `next` is `page->block_size` aligned? + if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) { + return _mi_block_next_is_corrupted(page,block,next); // returns NULL } return next; #else @@ -1101,6 +1304,8 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c #endif } + + /* ----------------------------------------------------------- arena blocks ----------------------------------------------------------- */ @@ -1141,17 +1346,40 @@ static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool commi return memid; } -static inline mi_memid_t _mi_memid_create_meta(void* mpage, size_t block_idx, size_t block_count) { - mi_memid_t memid = _mi_memid_create(MI_MEM_META); - memid.mem.meta.meta_page = mpage; - memid.mem.meta.block_index = (uint32_t)block_idx; - memid.mem.meta.block_count = (uint32_t)block_count; +static inline mi_memid_t _mi_memid_create_static(void* p, size_t size) { + mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; + memid.initially_committed = true; + memid.is_pinned = true; + return memid; +} + +static inline mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) { + mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; memid.initially_committed = true; - memid.initially_zero = true; + memid.initially_zero = iszero; memid.is_pinned = true; return memid; } +static inline size_t _mi_memid_size(mi_memid_t memid) { + if (mi_memid_is_os(memid)) { + return memid.mem.os.size; + } + else if (memid.memkind == MI_MEM_ARENA) { + return mi_size_of_slices(memid.mem.arena.slice_count); + } + else if (memid.memkind == MI_MEM_MALLOC) { + return memid.mem.malloc.size; + } + else { + mi_assert_internal(mi_memid_needs_no_free(memid)); + return 0; + } +} // ------------------------------------------------------------------- // Fast "random" shuffle @@ -1208,32 +1436,34 @@ static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) { #endif } -static inline void _mi_memcpy(void* dst, const void* src, size_t n) { +static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?) mi_rep_movsb(dst, src, n); + return dst; } else { - memcpy(dst, src, n); + return memcpy(dst, src, n); } } -static inline void _mi_memset(void* dst, int val, size_t n) { +static inline void* _mi_memset(void* dst, int val, size_t n) { if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127 mi_rep_stosb(dst, (uint8_t)val, n); + return dst; } else { - memset(dst, val, n); + return memset(dst, val, n); } } #else -static inline void _mi_memcpy(void* dst, const void* src, size_t n) { - memcpy(dst, src, n); +static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { + return memcpy(dst, src, n); } -static inline void _mi_memset(void* dst, int val, size_t n) { - memset(dst, val, n); +static inline void* _mi_memset(void* dst, int val, size_t n) { + return memset(dst, val, n); } #endif @@ -1246,42 +1476,40 @@ static inline void _mi_memset(void* dst, int val, size_t n) { #if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // On GCC/CLang we provide a hint that the pointers are word aligned. -static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { +static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); - _mi_memcpy(adst, asrc, n); + return _mi_memcpy(adst, asrc, n); } -static inline void _mi_memset_aligned(void* dst, int val, size_t n) { +static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - _mi_memset(adst, val, n); + return _mi_memset(adst, val, n); } #else // Default fallback on `_mi_memcpy` -static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { +static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - _mi_memcpy(dst, src, n); + return _mi_memcpy(dst, src, n); } -static inline void _mi_memset_aligned(void* dst, int val, size_t n) { +static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - _mi_memset(dst, val, n); + return _mi_memset(dst, val, n); } #endif -static inline void _mi_memzero(void* dst, size_t n) { - _mi_memset(dst, 0, n); +static inline void* _mi_memzero(void* dst, size_t n) { + return _mi_memset(dst, 0, n); } -static inline void _mi_memzero_aligned(void* dst, size_t n) { - _mi_memset_aligned(dst, 0, n); +static inline void* _mi_memzero_aligned(void* dst, size_t n) { + return _mi_memset_aligned(dst, 0, n); } - - #endif // MI_INTERNAL_H diff --git a/system/lib/mimalloc/include/mimalloc/prim-tls.h b/system/lib/mimalloc/include/mimalloc/prim-tls.h new file mode 100644 index 0000000000000..13f5860f95dad --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/prim-tls.h @@ -0,0 +1,424 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PRIM_TLS_H +#define MIMALLOC_PRIM_TLS_H + +#include "types.h" +#include "internal.h" // mi_decl_hidden + +// -------------------------------------------------------------------------- +// We need fast access to both a unique thread id (in `free.c:mi_free`) and +// to a thread-local theap pointer (in `alloc.c:mi_malloc`). +// +// For performance, we tend to use specialized code for various platforms. +// This leads to quite a few ifdefs but it is just for performance and there +// is always a portable fallback (based on regular thread local variables). +// +// Windows : use NtCurrentTeB and TlsAlloc (MI_TLS_MODEL_WIN32) +// Linux,FreeBSD : use thread locals with the initial-exec model (MI_TLS_MODEL_LOCAL) +// macOS : use pthread locals with assembly for the thread-id (MI_TLS_MODEL_PTHREADS) +// Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better? +// -------------------------------------------------------------------------- + +// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage +// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block +// static inline void mi_prim_tls_slot_set(size_t slot, void* value); + +static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread +static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap +static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api +static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap +static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap +static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached +static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) + + +// Default TLS model +#if !defined(MI_TLS_MODEL_LOCAL) && !defined(MI_TLS_MODEL_PTHREADS) && !defined(MI_TLS_MODEL_FIXED) && !defined(MI_TLS_MODEL_WIN32) +#if defined(_WIN32) +#define MI_TLS_MODEL_WIN32 1 +#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__) // and FreeBSD? +#define MI_TLS_MODEL_PTHREADS 1 +#else +#define MI_TLS_MODEL_LOCAL 1 +#endif +#endif + + +//------------------------------------------------------------------- +// Access to the thread pointer +//------------------------------------------------------------------- + +// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id +// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) +// Nevertheless, it seems needed on older graviton platforms (see issue #851). +// For now, we only enable this for specific platforms. +#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */ + #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \ + && !defined(__CYGWIN__) \ + && !defined(MI_LIBC_MUSL) \ + && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */ + #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ + || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ + || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv))) + #define MI_USE_BUILTIN_THREAD_POINTER 1 + #endif + #endif +#endif + +#if defined(MI_PRIM_THREAD_POINTER) // potential user override +static inline void** mi_prim_thread_pointer(void) { + return MI_PRIM_THREAD_POINTER(); +} +#elif defined(_WIN32) +static inline void** mi_prim_thread_pointer(void) { + return (void**)NtCurrentTeb(); +} +#elif MI_USE_BUILTIN_THREAD_POINTER +static inline void** mi_prim_thread_pointer(void) { + return (void**)__builtin_thread_pointer(); +} +#elif defined(__GNUC__) && !defined(__CYGWIN__) + #if defined(__aarch64__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) + __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + return tcb; + } + #elif defined(__riscv) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ ("mv %0, tp" : "=r" (tcb)); + return tcb; + } + #elif defined(__arm__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb)); + return tcb; + } + #elif defined(__i386__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ ("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS + return tcb; + } + #elif defined(__x86_64__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + #if defined(__APPLE__) + __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS + #elif (MI_INTPTR_SIZE==4) + __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI + #else + __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS + #endif + return tcb; + } + #else + #define MI_NO_THREAD_POINTER (1) + #endif +#elif MI_USE_PTHREADS && defined(__APPLE__) +static inline void** mi_prim_thread_pointer(void) { + return (void**)pthread_self(); +} +#else +#define MI_NO_THREAD_POINTER (1) +#endif + +#if !MI_NO_THREAD_POINTER +#define MI_HAS_TLS_SLOT (1) +static inline void* mi_prim_tls_slot(size_t slot) { + #if defined(_WIN32) + #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) + return (void*)__readgsqword((unsigned long)(slot*sizeof(void*))); // direct load at offset from gs + #elif _M_IX86 && !defined(_M_ARM64EC) + return (void*)__readfsdword((unsigned long)(slot*sizeof(void*))); // direct load at offset from fs + #else + return mi_prim_thread_pointer()[slot]; + #endif + #else + return mi_prim_thread_pointer()[slot]; + #endif +} + +static inline void mi_prim_tls_slot_set(size_t slot, void* value) { + mi_prim_thread_pointer()[slot] = value; +} +#endif + +/* ---------------------------------------------------------------------------------------- + Get a unique thread-id for a thread +---------------------------------------------------------------------------------------- */ + +// Get a unique id for the current thread. +#if defined(MI_PRIM_THREAD_ID) +static inline mi_threadid_t __mi_prim_thread_id(void) { + return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) +} +#elif !MI_NO_THREAD_POINTER +static inline mi_threadid_t __mi_prim_thread_id(void) { + #if defined(__BIONIC__) + return (mi_threadid_t)mi_prim_tls_slot(1); + #else + return (mi_threadid_t)mi_prim_thread_pointer(); + #endif +} +#else +// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). +extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper; +static inline mi_threadid_t __mi_prim_thread_id(void) { + return (uintptr_t)&__mi_thread_id_helper; +} +#endif + +static inline mi_threadid_t _mi_prim_thread_id(void) { + const mi_threadid_t tid = __mi_prim_thread_id(); + mi_assert_internal(tid > MI_THREADID_DETACHED); + mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear? + return tid; +} + + +/* ---------------------------------------------------------------------------------------- +Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`). + +This is inlined here as it is on the fast path for allocation functions. +We have 4 models: + +- MI_TLS_MODEL_LOCAL: use regular thread local (default on Linux, FreeBSD, etc) + On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a + thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures + that the storage will always be available and properly initialized (with an empty theap). + + On some platforms the underlying TLS implementation (or the loader) will call itself `malloc` + on a first access to a thread local and recurse in the MI_TLS_MODEL_LOCAL. + A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process + is initialized before accessing the thread-local. This is a check in the fast path though + so this should be avoided. + +- MI_TLS_MODEL_PTHREADS: use `pthread_getspecific`. (default on macOS and OpenBSD, maybe good for Android as well?) + Use pthread local storage. Can be as fast as thread locals on many platforms (like recent macOS). + +- MI_TLS_MODEL_FIXED: use a fixed slot in the TLS block. + This reserves an unused and fixed TLS slot. This is fast and avoids the problem + where the underlying TLS implementation (or the loader) will call itself `malloc` + on a first access to a thread local (and recurse in the MI_TLS_MODEL_LOCAL). + This goes wrong though if the OS or a library uses the same fixed slot, and also + prevents multiple instances of mimalloc in the same process. + +- MI_TLS_MODEL_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows) + We use TlsAlloc'd slot. First tries to use one of the "direct" first 64 slots which + are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots). + (If the allocated slot happens to always be under 64 for a particular program, + one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.) + +Each model should define `MI_THEAP_INITASNULL` to signify that the initial value +returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap). +This incurs an extra check in the fast path (but can often be combined in an existing check). +------------------------------------------------------------------------------------------- */ + +#if !defined(MI_TLS_RECURSE_GUARD) && MI_TLS_MODEL_LOCAL && defined(__APPLE__) +#define MI_TLS_RECURSE_GUARD 1 // macOS can allocate on thread-local initialization +#endif + +// Declared this way to optimize register spills and branches +mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void); + +static inline mi_theap_t* __mi_theap_empty(void) { + #if __GNUC__ + __asm(""); // prevent conditional load + return (mi_theap_t*)&_mi_theap_empty; + #else + return _mi_theap_empty_get(); + #endif +} + +#if MI_TLS_MODEL_LOCAL +// Thread local with an initial value (default on Linux). Very efficient. +extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from +extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap + +// defined in `init.c`; do not use these directly +extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? + +static inline mi_theap_t* _mi_theap_default(void) { + #if defined(MI_TLS_RECURSE_GUARD) + if mi_unlikely(!_mi_process_is_initialized) return _mi_theap_empty_get(); + #endif + return __mi_theap_default; +} + +static inline mi_theap_t* _mi_theap_cached(void) { + return __mi_theap_cached; +} + +#elif MI_TLS_MODEL_PTHREADS +// Dynamic pthread slots. This can be fast depending on the platform (default for macOS and OpenBSD) +// On some platforms (like macOS), the loader might allocate on thread local declarations which +// can be avoided with pthreads. +#define MI_THEAP_INITASNULL 1 + +extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key; +extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key; + +static inline mi_theap_t* _mi_theap_default(void) { + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); + #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT + // on apple arm64, the pthread specific slots are direct slots; inline it to avoid a stack frame setup in `mi_malloc` + // todo: this is probably also the case on x64 and power pc? + if (key == MI_PTHREAD_KEY_INVALID) return NULL; + return (mi_theap_t*)mi_prim_tls_slot(key); + #else + return (mi_theap_t*)mi_pthread_key_get(key); + #endif +} + +static inline mi_theap_t* _mi_theap_cached(void) { + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key); + #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT + if (key == MI_PTHREAD_KEY_INVALID) return NULL; + return (mi_theap_t*)mi_prim_tls_slot(key); + #else + return (mi_theap_t*)mi_pthread_key_get(key); + #endif +} + +#elif MI_TLS_MODEL_WIN32 +// Dynamic TLS slots -- this is the default on Windows. +#define MI_THEAP_INITASNULL 1 + +// We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available) +// See for the offsets. +#if MI_SIZE_SIZE==4 +#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_INTPTR_SIZE) +#else +#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_INTPTR_SIZE) +#endif + +extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot; + +static inline mi_theap_t* _mi_theap_default(void) { + const size_t slot = mi_atomic_load_relaxed(&_mi_theap_default_slot); + mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); + #if !MI_WIN_DIRECT_TLS + if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? + mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is actually the expansion slot entry + if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) + theap = eslots[mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot)]; + } + } + #endif + return theap; +} + +static inline mi_theap_t* _mi_theap_cached(void) { + const size_t slot = mi_atomic_load_relaxed(&_mi_theap_cached_slot); + mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); + #if !MI_WIN_DIRECT_TLS + if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? + mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry + if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) + theap = eslots[mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot)]; + } + } + #endif + return theap; +} + +#elif MI_TLS_MODEL_FIXED +// Fixed TLS slot. Can be the fastest approach, but does not work if there are multiple instances of +// mimalloc in the same process. Most OS's do not have official user reserved fixed slots so this cannot be +// guaranteed to work in general. +#define MI_THEAP_INITASNULL 1 + +#if !MI_HAS_TLS_SLOT +#error this platform cannot support MI_TLS_MODEL_FIXED without defining mi_prim_tls_slot +#endif + +#if !defined(MI_TLS_MODEL_FIXED_DEFAULT) + #if defined(__APPLE__) && !defined(__POWERPC__) // macOS on arm64 or x64 + // we use the last two swift framework slots which seem unused. + // we may want to use slot 6 and 11 instead which are only used by Windows emulation. + // see for assigned slots + #define MI_TLS_MODEL_FIXED_DEFAULT 108 + #define MI_TLS_MODEL_FIXED_CACHED 109 + #elif defined(_WIN32) + // we use two seemingly unused fields in the Windows TEB. + // see + #define MI_TLS_MODEL_FIXED_DEFAULT 5 // arbitrary user pointer + #define MI_TLS_MODEL_FIXED_CACHED 7 // environment pointer (used by OS2) + #else + #error define the TLS model fixed slots (or change the TLS model away from MI_TLS_MODEL_FIXED) + #endif +#endif + +static inline mi_theap_t* _mi_theap_default(void) { + return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_DEFAULT); +} + +static inline mi_theap_t* _mi_theap_cached(void) { + return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_CACHED); +} + +#else +#error "no TLS model is defined for this platform?" +#endif + + +// Check if a thread is initialized (without using a thread-local if using fixed slots) +static inline bool _mi_thread_is_initialized(void) { + return mi_theap_is_initialized(_mi_theap_default()); +} + +// Get (and possible create) the theap belonging to a heap +// We cache the last accessed theap in `_mi_theap_cached` for better performance. +static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + return _mi_heap_theap_get_or_init(heap); +} + +// Get the theap belonging to a heap without creating it if it is not yet initialized. +static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); // don't update the cache on a query + mi_assert_internal(theap==NULL || (!_mi_is_empty_theap(theap) && theap->heap==heap)); + return theap; +} + +// Find the associated theap or NULL if it does not exist (during shutdown) +// Should be fast as it is called in `free.c:mi_free_try_collect`. +static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { + mi_heap_t* const heap = mi_page_heap(page); + mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); + if (theap==NULL) return NULL; + if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation + mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap)); + // note: for pages allocated by a detached theap, the returned theap may not be detached + return theap; +} + +#endif // MI_PRIM_TLS_H diff --git a/system/lib/mimalloc/include/mimalloc/prim.h b/system/lib/mimalloc/include/mimalloc/prim.h index a58fb2735188c..82b2d9c3d242e 100644 --- a/system/lib/mimalloc/include/mimalloc/prim.h +++ b/system/lib/mimalloc/include/mimalloc/prim.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file #pragma once #ifndef MIMALLOC_PRIM_H #define MIMALLOC_PRIM_H -#include "internal.h" // mi_decl_hidden + +#include "types.h" // -------------------------------------------------------------------------- // This file specifies the primitive portability API. @@ -23,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file // OS memory configuration typedef struct mi_os_mem_config_s { size_t page_size; // default to 4KiB - size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows) + size_t large_page_size; // 0 if not supported, usually 2MiB size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) size_t physical_memory_in_kib; // physical memory size in KiB size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization) @@ -128,435 +129,8 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap); // Is this thread part of a thread pool? bool _mi_prim_thread_is_in_threadpool(void); -// Yield to other threads. Should be similar to `sleep(0)`. +// Yield to other threads. Should be similar to `sleep(0)`. // Is called only in rare situations and does not have to be lightning fast. void _mi_prim_thread_yield(void); -//------------------------------------------------------------------- -// Access to TLS (thread local storage) slots. -// We need fast access to both a unique thread id (in `free.c:mi_free`) and -// to a thread-local theap pointer (in `alloc.c:mi_malloc`). -// To achieve this we use specialized code for various platforms. -//------------------------------------------------------------------- - -// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot. -// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform. -// If you test on another platform and it works please send a PR :-) -// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. -// -// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly, -// but unfortunately we can not detect support reliably (see issue #883) -// We also use it on Apple OS as we use a TLS slot for the default theap there. -#if (defined(_WIN32)) || \ - (defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__) || defined(__riscv))) \ - || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \ - || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ - || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ - || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ - )) - -static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { - void* res; - const size_t ofs = (slot*sizeof(void*)); - #if defined(_WIN32) - #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) - res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs - #elif _M_IX86 && !defined(_M_ARM64EC) - res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs - #else - res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs); - #endif - #elif defined(__i386__) - __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS - #elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS - #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI - #elif defined(__x86_64__) - __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - res = tcb[slot]; - #elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); - #endif - res = tcb[slot]; - #elif defined(__riscv) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mv %0, tp" : "=r" (tcb)); - res = tcb[slot]; - #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 - MI_UNUSED(ofs); - res = pthread_getspecific(slot); - #else - #define MI_HAS_TLS_SLOT 0 - MI_UNUSED(ofs); - res = NULL; - #endif - return res; -} - -#ifndef MI_HAS_TLS_SLOT -#define MI_HAS_TLS_SLOT 1 -#endif - -// setting a tls slot is only used on macOS for now -static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { - const size_t ofs = (slot*sizeof(void*)); - #if defined(_WIN32) - ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs); - #elif defined(__i386__) - __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS - #elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS - #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI - #elif defined(__x86_64__) - __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - tcb[slot] = value; - #elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); - #endif - tcb[slot] = value; - #elif defined(__riscv) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mv %0, tp" : "=r" (tcb)); - tcb[slot] = value; - #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 - MI_UNUSED(ofs); - pthread_setspecific(slot, value); - #else - MI_UNUSED(ofs); MI_UNUSED(value); - #endif -} - -#endif - - -// defined in `init.c`; do not use these directly -extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main; // theap belonging to the main heap -extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? - - -//------------------------------------------------------------------- -// Get a fast unique thread id. -// -// Getting the thread id should be performant as it is called in the -// fast path of `_mi_free` and we specialize for various platforms as -// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. -// We only require _mi_prim_thread_id() to return a unique id -// for each thread (unequal to zero) with the bottom 2 bits clear. -//------------------------------------------------------------------- - -// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id -// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) -// Nevertheless, it seems needed on older graviton platforms (see issue #851). -// For now, we only enable this for specific platforms. -#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */ - #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \ - && !defined(__CYGWIN__) \ - && !defined(MI_LIBC_MUSL) \ - && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */ - #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ - || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ - || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ - || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__))) - #define MI_USE_BUILTIN_THREAD_POINTER 1 - #endif - #endif -#endif - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept; - -static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { - const mi_threadid_t tid = __mi_prim_thread_id(); - mi_assert_internal(tid > 1); - mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear? - return tid; -} - -// Get a unique id for the current thread. -#if defined(MI_PRIM_THREAD_ID) - -static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { - const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) - mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits - return tid; -} - -#elif defined(_WIN32) - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - // Windows: works on Intel and ARM in both 32- and 64-bit - return (uintptr_t)NtCurrentTeb(); -} - -#elif MI_USE_BUILTIN_THREAD_POINTER - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - // Works on most Unix based platforms with recent compilers - return (uintptr_t)__builtin_thread_pointer(); -} - -#elif MI_HAS_TLS_SLOT - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - #if defined(__BIONIC__) - // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id - // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 - return (uintptr_t)mi_prim_tls_slot(1); - #else - // in all our other targets, slot 0 is the thread id - // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h - // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 - return (uintptr_t)mi_prim_tls_slot(0); - #endif -} - -#else - -// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - return (uintptr_t)&__mi_theap_main; -} - -#endif - - - -/* ---------------------------------------------------------------------------------------- -Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`). - -This is inlined here as it is on the fast path for allocation functions. -We have 4 models: - -- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc) - On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a - thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures - that the storage will always be available and properly initialized (with an empty theap). - - On some platforms the underlying TLS implementation (or the loader) will call itself `malloc` - on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL. - A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process - is initialized before accessing the thread-local. This is a check in the fast path though - so this should be avoided. - -- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS) - This reserves an unused and fixed TLS slot. This is fast and avoids the problem - where the underlying TLS implementation (or the loader) will call itself `malloc` - on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL). - This goes wrong though if the OS or a library uses the same fixed slot. - -- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows) - Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which - can be more efficient. First tries to use one of the "direct" first 64 slots which - are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots). - (If the allocated slot happens to always be under 64 for a particular program, - one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.) - -- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?) - Use pthread local storage. Somewhat slow but can work well depending on the platform. - -Each model should define `MI_THEAP_INITASNULL` to signify that the initial value -returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap). -This incurs an extra check in the fast path (but can often be combined in an existing check). -------------------------------------------------------------------------------------------- */ - -static inline mi_theap_t* _mi_theap_default(void); -static inline mi_theap_t* _mi_theap_cached(void); - -// Default TLS model -#if !defined(MI_TLS_MODEL_THREAD_LOCAL) && !defined(MI_TLS_MODEL_DYNAMIC_PTHREADS) - #if defined(_WIN32) - #define MI_TLS_MODEL_DYNAMIC_WIN32 1 - #elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64 - // #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a tad slower - #define MI_TLS_MODEL_FIXED_SLOT 1 - // we use the last 2 7-bit slots which seem unused. - // @apple: it would be great to get 2 official slots for custom allocators :-) - #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 126 - #define MI_TLS_MODEL_FIXED_SLOT_CACHED 127 - // see for assigned slots - #elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__) - #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 - // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1 - #else - #define MI_TLS_MODEL_THREAD_LOCAL 1 - #endif -#endif - -#if !defined(MI_TLS_RECURSE_GUARD) && MI_TLS_MODEL_THREAD_LOCAL && defined(__APPLE__) -#define MI_TLS_RECURSE_GUARD 1 // macOS can allocate on thread-local initialization -#endif - -// Declared this way to optimize register spills and branches -mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void); - -static inline mi_theap_t* __mi_theap_empty(void) { - #if __GNUC__ - __asm(""); // prevent conditional load - return (mi_theap_t*)&_mi_theap_empty; - #else - return _mi_theap_empty_get(); - #endif -} - -#if MI_TLS_MODEL_THREAD_LOCAL -// Thread local with an initial value (default on Linux). Very efficient. - -extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from -extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap - -static inline mi_theap_t* _mi_theap_default(void) { - #if defined(MI_TLS_RECURSE_GUARD) - if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get(); - #endif - return __mi_theap_default; -} - -static inline mi_theap_t* _mi_theap_cached(void) { - return __mi_theap_cached; -} - -#elif MI_TLS_MODEL_FIXED_SLOT -// Fixed TLS slot (default on macOS). -#define MI_THEAP_INITASNULL 1 - -static inline mi_theap_t* _mi_theap_default(void) { - return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT); -} - -static inline mi_theap_t* _mi_theap_cached(void) { - return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED); -} - -#elif MI_TLS_MODEL_DYNAMIC_WIN32 -// Dynamic TLS slot (default on Windows) -#define MI_THEAP_INITASNULL 1 - -// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available) -// See for the offsets. -#if MI_SIZE_SIZE==4 -#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE) -#else -#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE) -#endif - -extern mi_decl_hidden size_t _mi_theap_default_slot; -extern mi_decl_hidden size_t _mi_theap_cached_slot; -extern mi_decl_hidden size_t _mi_theap_default_expansion_slot; -extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot; - -static inline mi_theap_t* _mi_theap_default(void) { - const size_t slot = _mi_theap_default_slot; - mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); - #if !MI_WIN_DIRECT_TLS - if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? - mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry - if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) - theap = eslots[_mi_theap_default_expansion_slot]; - } - } - #endif - return theap; -} - -static inline mi_theap_t* _mi_theap_cached(void) { - const size_t slot = _mi_theap_cached_slot; - mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); - #if !MI_WIN_DIRECT_TLS - if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? - mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry - if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) - theap = eslots[_mi_theap_cached_expansion_slot]; - } - } - #endif - return theap; -} - -#elif MI_TLS_MODEL_DYNAMIC_PTHREADS -// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD) -#define MI_THEAP_INITASNULL 1 - -extern mi_decl_hidden pthread_key_t _mi_theap_default_key; -extern mi_decl_hidden pthread_key_t _mi_theap_cached_key; - -static inline mi_theap_t* _mi_theap_default(void) { - #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL - // we can skip this check if using the initial key will return NULL from pthread_getspecific - if mi_unlikely(_mi_theap_default_key==0) { return NULL; } - #endif - return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key); -} - -static inline mi_theap_t* _mi_theap_cached(void) { - #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL - // we can skip this check if using the initial key will return NULL from pthread_getspecific - if mi_unlikely(_mi_theap_cached_key==0) { return NULL; } - #endif - return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key); -} - -#else -#error "no TLS model is defined for this platform?" -#endif - - -// Check if a thread is initialized (without using a thread-local if using fixed slots) -static inline bool _mi_thread_is_initialized(void) { - return (mi_theap_is_initialized(_mi_theap_default())); -} - -// Get (and possible create) the theap belonging to a heap -// We cache the last accessed theap in `_mi_theap_cached` for better performance. -static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) { - mi_theap_t* theap = _mi_theap_cached(); - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap; - #else - if mi_likely(_mi_theap_heap(theap)==heap) return theap; - #endif - return _mi_heap_theap_get_or_init(heap); -} - -// Get the theap belonging to a heap without creating it if it is not yet initialized. -static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { - mi_theap_t* theap = _mi_theap_cached(); - #if MI_THEAP_INITASNULL - if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap) - #else - if mi_unlikely(_mi_theap_heap(theap)!=heap) - #endif - { - theap = _mi_heap_theap_get_peek(heap); // don't update the cache on a query (?) - } - mi_assert(theap==NULL || _mi_theap_heap(theap)==heap); - return theap; -} - -// Find the associated theap or NULL if it does not exist (during shutdown) -// Should be fast as it is called in `free.c:mi_free_try_collect`. -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { - mi_heap_t* const heap = page->heap; - mi_theap_t* theap; - if mi_likely(heap==NULL) { theap = __mi_theap_main; } // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)! - else { theap = _mi_heap_theap_peek(heap); } - mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id); - return theap; -} - #endif // MI_PRIM_H diff --git a/system/lib/mimalloc/include/mimalloc/track.h b/system/lib/mimalloc/include/mimalloc/track.h index 1f40212978922..753363366be2e 100644 --- a/system/lib/mimalloc/include/mimalloc/track.h +++ b/system/lib/mimalloc/include/mimalloc/track.h @@ -135,16 +135,16 @@ defined, undefined, or not accessible at all: #if MI_PADDING #define mi_track_malloc(p,reqsize,zero) \ - if ((p)!=NULL) { \ + do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)==(reqsize)); \ mi_track_malloc_size(p,reqsize,reqsize,zero); \ - } + } } while(0) #else #define mi_track_malloc(p,reqsize,zero) \ - if ((p)!=NULL) { \ + do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ - } + } } while(0) #endif #endif // MI_TRACK_H diff --git a/system/lib/mimalloc/include/mimalloc/types.h b/system/lib/mimalloc/include/mimalloc/types.h index d67f0a1a81611..308c17bd3deae 100644 --- a/system/lib/mimalloc/include/mimalloc/types.h +++ b/system/lib/mimalloc/include/mimalloc/types.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -55,16 +55,18 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_STAT 1 // Define MI_SECURE to enable security mitigations -// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption -// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages) -// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) -// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`) -// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`) +// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption +// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages +// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) +// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`) +// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks. #if !defined(MI_SECURE) #define MI_SECURE 0 #endif +#define MI_PADDING 0 + // Define MI_DEBUG for assertion and invariant checking // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON) // #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON) @@ -87,19 +89,19 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) -#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL +#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL #define MI_GUARDED 1 #endif // Reserve extra padding at the end of each block to be more resilient against theap block overflows. -// The padding can detect buffer overflow on free. +// The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. #if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 #endif -// Check padding bytes; allows byte-precise buffer overflow detection -#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1) -#define MI_PADDING_CHECK 1 +// Check for byte-precise buffer overflow? +#if !defined(MI_PADDING_CHECK_BYTES) && MI_PADDING && (MI_SECURE>=5 || MI_DEBUG>=1) +#define MI_PADDING_CHECK_BYTES 1 #endif @@ -109,6 +111,17 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif +// Deprecated (double-free is now checked with padding) +// #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +// #define MI_CHECK_DOUBLE_FREE 1 +// #endif + +#if MI_SECURE>=4 || MI_PADDING || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP) +#define MI_PAGE_KEY_COUNT 2 +#else +#define MI_PAGE_KEY_COUNT 1 +#endif + // Enable large pages for objects between 64KiB and 512KiB. // This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB // are quite random which can lead to too many partially used large pages (but see issue #1104). @@ -119,7 +132,6 @@ terms of the MIT license. A copy of the license can be found in the file // Place page meta info at the start of the page area or keep it separate? // Separate keeps the page info at the arena start (default) which is more secure // and reduces wasted space due to alignment and block sizes. -// (but also reserves more memory up front (about 2MiB per GiB)) #if !defined(MI_PAGE_META_IS_SEPARATED) #if MI_PAGE_MAP_FLAT #define MI_PAGE_META_IS_SEPARATED 0 @@ -128,19 +140,38 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif +// We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS +// This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. +// This only works if valid pointers are passed to `mi_free` though. However, checked +// free `mi_cfree` can still use the page map to validate pointers. +#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP +#if MI_PAGE_META_IS_SEPARATED +#define MI_PAGE_META_IS_ALIGNED 1 +#define MI_PAGE_META_ALIGNED_CHUNKS MI_INTPTR_SIZE +#else +#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" +#endif +#endif + // We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). -#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) -#define MI_PAGE_META_ALIGNED_FREE_SMALL 0 +#if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // cannot be guarded as that may allocate large blocks for small allocations +#define MI_PAGE_META_SMALL_IS_ALIGNED 1 +#else +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#endif #endif // Configuration checks #if !MI_PAGE_META_IS_SEPARATED && MI_SECURE #error "secure mode should use separated page infos" #endif -#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE -#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL" +#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE +#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)" #endif #if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT #error "cannot have a flat page map with separated page infos" @@ -149,7 +180,6 @@ terms of the MIT license. A copy of the license can be found in the file #warning "mimalloc assertions enabled in a release build" #endif - // -------------------------------------------------------------- // Sizes of internal data-structures // (comments specify sizes on 64-bit, usually 32-bit is halved) @@ -159,14 +189,14 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MI_ARENA_SLICE_SHIFT #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT - #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64 + #elif MI_SECURE>=5 && ((__APPLE__ && MI_ARCH_ARM64) || (defined(PAGE_SIZE) && PAGE_SIZE >= 16*MI_KiB)) #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages #else #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) #endif #endif -#if MI_ARENA_SLICE_SHIFT < 12 -#error Arena slices should be at least 4KiB +#if MI_ARENA_SLICE_SHIFT < 13 +#error Arena slices should be at least 8KiB #endif #ifndef MI_BCHUNK_BITS_SHIFT @@ -182,6 +212,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits #define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB #define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE) +#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) #define MI_ARENA_MIN_OBJ_SLICES (1) #define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit) @@ -206,11 +237,18 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BIN_COUNT (MI_BIN_FULL+1) // We never allocate more than PTRDIFF_MAX (see also ) -#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX +#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX -// Minimal commit for a page on-demand commit (should be >= OS page size) -#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE +// Minimal commit for a page on-demand commit +#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ +#if MI_PAGE_META_IS_ALIGNED +#define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS) +#define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) +#define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT +#else +#define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN +#endif // ------------------------------------------------------ // Arena's are large reserved areas of memory allocated from @@ -248,15 +286,14 @@ typedef struct mi_subproc_s mi_subproc_t; // Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated. // The memid keeps track of this. typedef enum mi_memkind_e { - MI_MEM_NONE, // not allocated + MI_MEM_NONE, // not allocated (or static) MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`)) - MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`) MI_MEM_OS, // allocated from the OS MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory) MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`) - MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps) + MI_MEM_MALLOC // allocated with mi_malloc } mi_memkind_t; static inline bool mi_memkind_is_os(mi_memkind_t memkind) { @@ -267,6 +304,7 @@ static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) { return (memkind <= MI_MEM_STATIC); } +typedef struct mi_meta_page_s mi_meta_page_t; typedef struct mi_memid_os_info { void* base; // actual base address of the block (used for offset aligned allocations) @@ -280,17 +318,16 @@ typedef struct mi_memid_arena_info { uint32_t slice_count; // allocated slices } mi_memid_arena_info_t; -typedef struct mi_memid_meta_info { - void* meta_page; // meta-page that contains the block - uint32_t block_index; // block index in the meta-data page - uint32_t block_count; // allocated blocks -} mi_memid_meta_info_t; +typedef struct mi_memid_malloc_info { + void* base; // returned pointer + size_t size; // allocated size +} mi_memid_malloc_info_t; typedef struct mi_memid_s { union { - mi_memid_os_info_t os; // only used for MI_MEM_OS - mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA - mi_memid_meta_info_t meta; // only used for MI_MEM_META + mi_memid_os_info_t os; // only used for MI_MEM_OS(_HUGE/_REMAP) + mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA + mi_memid_malloc_info_t malloc; // only used for MI_MEM_MALLOC } mem; mi_memkind_t memkind; bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages) @@ -298,6 +335,8 @@ typedef struct mi_memid_s { bool initially_zero; // `true` if the memory was originally zero initialized } mi_memid_t; +#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ } +#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC) static inline bool mi_memid_is_os(mi_memid_t memid) { return mi_memkind_is_os(memid.memkind); @@ -335,6 +374,7 @@ typedef struct mi_block_s { #define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01) #define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02) #define MI_PAGE_FLAG_MASK MI_ZU(0x03) +#define MI_PAGE_FLAG_BITS (2) typedef size_t mi_page_flags_t; // There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue), @@ -342,7 +382,8 @@ typedef size_t mi_page_flags_t; // in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse. // Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked) #define MI_THREADID_ABANDONED MI_ZU(0) -#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1) +#define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS) +#define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS) // Thread free list. // Points to a list of blocks that are freed by other threads. @@ -351,6 +392,9 @@ typedef size_t mi_page_flags_t; // This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; +// Convenience +typedef size_t mi_used_t; + // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: // `free` for blocks that can be allocated, @@ -376,36 +420,39 @@ typedef uintptr_t mi_thread_free_t; // that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that // are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages). // - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`. -// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` -// - Using `uint16_t` does not seem to slow things down +// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` -typedef struct mi_page_s { +typedef struct mi_page_s { + #if (MI_PAGE_META_IS_ALIGNED) + _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices) + #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) - mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - uint16_t used; // number of blocks in use (including blocks in `thread_free`) + mi_used_t used; // number of blocks in use (including blocks in `thread_free`) + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + size_t block_size; // const: size available in each block (always `>0`) + size_t page_offset; // const: relative offset from the page to the start of the blocks uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory + uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) uint8_t retire_expire; // expiration count for retired blocks bool free_is_zero; // `true` if the blocks in the free list are zero initialized - - mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + // next cache line _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) - - size_t block_size; // const: size available in each block (always `>0`) - uint32_t page_woffset; // const: offset relative to the page (in machine words) to the start of the blocks - uint32_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already) - - #if (MI_ENCODE_FREELIST || MI_PADDING) - uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary - #endif - mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) mi_heap_t* heap; // const: the heap owning this page struct mi_page_s* next; // next page owned by the theap with the same `block_size` struct mi_page_s* prev; // previous page owned by the theap with the same `block_size` mi_memid_t memid; // const: provenance of the page memory + + #if (MI_ENCODE_FREELIST || MI_PADDING) + uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary + // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + // uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align + #endif } mi_page_t; @@ -413,7 +460,7 @@ typedef struct mi_page_s { // Object sizes // ------------------------------------------------------ -#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map. +#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // page area's must be aligned on this for the page map. #define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b) #define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment) #define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses. @@ -498,7 +545,7 @@ typedef struct mi_random_cxt_s { #if MI_PADDING typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) - uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) + uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes) } mi_padding_t; #define MI_PADDING_SIZE (sizeof(mi_padding_t)) #define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) @@ -512,12 +559,15 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { + // put in front for fast small allocations + mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. + _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) _Atomic(size_t) refcount; // reference count - _Atomic(size_t) freed; // ensure atomic free-ing + unsigned long long heartbeat; // monotonic heartbeat count - uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) @@ -534,13 +584,13 @@ struct mi_theap_s { long page_full_retain; // how many full pages can be retained per queue (before abandoning them) bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint + bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` #if MI_GUARDED size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) size_t guarded_sample_count; // current sample count (counting down to 0) #endif - mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics @@ -583,7 +633,7 @@ typedef struct mi_heap_s { _Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL) mi_lock_t arena_pages_lock; // lock to update the arena_pages array - + mi_memid_t memid; // provenance of the heap memory mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap } mi_heap_t; @@ -601,6 +651,7 @@ struct mi_subproc_s { size_t subproc_seq; // unique id for sub-processes mi_subproc_t* next; // list of all sub-processes mi_subproc_t* prev; + _Atomic(mi_meta_page_t*) meta_pages; // meta data pages _Atomic(size_t) arena_count; // current count of arena's _Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process @@ -612,12 +663,16 @@ struct mi_subproc_s { mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; + mi_theap_t* theap_meta; // detached theap for allocating meta-data + mi_lock_t theap_meta_lock; // all allocations in theap_meta need a lock + _Atomic(size_t) thread_count; // current threads associated with this sub-process _Atomic(size_t) thread_total_count; // total created threads associated with this sub-process _Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|) _Atomic(size_t) heap_total_count; // total created heaps in this sub-process mi_memid_t memid; // provenance of this memory block (meta or static) + mi_subproc_t* parent; // subproc in which this one was allocated mi_decl_align(8) // needed on some 32-bit platforms mi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed, // and otherwise merged with heap stats when those are deleted @@ -659,16 +714,16 @@ struct mi_tld_s { #define MI_ARENA_BIN_COUNT (MI_MAX_SINGLETON_BIN+1) #define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit) -#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) +#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) // 16 GiB typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`) typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`) -typedef struct mi_arena_pages_s { +struct mi_arena_pages_s { mi_bitmap_t* pages; // all registered pages (abandoned and owned) mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page) // followed by the bitmaps (whose siz`es depend on the arena size) -} mi_arena_pages_t; +}; // A memory arena @@ -676,6 +731,7 @@ typedef struct mi_arena_s { mi_memid_t memid; // provenance of the memory area mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`) size_t arena_idx; // index in the arenas array + void* start; // actual start of the arena area (the arena_s info may come later due to guard pages etc.) size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`) size_t info_slices; // initial slices reserved for the arena bitmaps diff --git a/system/lib/mimalloc/readme.md b/system/lib/mimalloc/readme.md index 374b6d48e7fb5..dcfc524b1abe3 100644 --- a/system/lib/mimalloc/readme.md +++ b/system/lib/mimalloc/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.4.1` (2026-07-14) recommended. -Latest v2 release: `v2.4.1` (2026-07-14) stable. -Latest v1 release: `v1.9.11` (2026-07-14) legacy. +Latest release : `v3.5.1` (2026-09-01) recommended. +Latest v2 release: `v2.5.1` (2026-09-01) stable, legacy. +Latest v1 release: `v1.15.1` (2026-09-01) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -84,10 +84,35 @@ New development is mostly on v3, while v1 and v2 are maintained with security an (much) less memory. Also supports true first-class heaps (that can allocate from any thread) and has more efficient heap-walking (for the CPython GC for example). (release tags: `v3.x`, development branch `dev3`). -- __v2__: stable mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) -- __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). Send PR's against this version if possible. +- __v2__: stable legacy mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) +- __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). + __Send PR's against this version if possible.__ ### Releases + +* 2026-09-01: Added a readme section on [getting the best performance](#getting-the-best-performance). +* 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. + (v3) Added `mi_free_small(_nonnull)` for runtimes, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. + (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. + (v3) Default to `-DMI_ALLOW_THP=FULL` on Linux, where we never split transparent huge pages (THP) during purging (#1282). This + may increase rss but can also improve performance. Use `-DMI_ALLOW_THP=OFF` to disable the use of THP. + (v3) Improved cache behavior for small allocations. Improved riscV suppor, add riscV to CI. Various small build fixes. +* 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` + using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for + faster load-acquire, increase retired page count from 1 to 3, faster double free + detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan + Andrade, #1346), faster pagemap lookup. + Other: fix numa sparse node count detection (#1365), add theap stats retrieval, + use __builtin_thread_pointer on riscV (#1363), fix C mode compilation on x86 with + msvc (#1361), improved mingw-ucrt64 support. +* 2026-08-05, `v1.9.15`, `v2.4.5`, `v3.4.5`: Fix compilation with xmalloc (#1353), make the build deterministic (#1355), + mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper + lock backoff for first-class heap deletion, improved riscv64 codegen. +* 2026-08-01, `v1.9.14`, `v2.4.4`, `v3.4.4`: various bug and security fixes through Opus 5 LLM audit (issue #1271, by @Zoxc). + (v3): use pthreads by default on macOS (issue #1333, issue #1327), fix glibc 2.44 crash (issue #1341), fix alignment check for realloc_aligned. (v1,v2,v3): Add initial mingw support, + set `errno` on allocation errors, improved double-free checks and size checks in secure mode, enable + build for universal windows platform and xbox (pr #1340), fix numa node detection when numa nodes are sparse. +* 2026-07-20, `v3.4.3`: revert TLS slots on macOS to 108/109 (issue #1333). * 2026-07-14, `v1.9.11`, `v2.4.1`, `v3.4.1`: various bug and security fixes through LLM audit (by @Zoxc). Fix issue with using OS memory instead of arenas for > 4GiB memory usage (v3), fix concurrency bug in concurrent heap destroy (v3), detect riscV virtual address bits at runtime, add riscV TLS support, reduce spinlock waits (v2), @@ -121,24 +146,6 @@ New development is mostly on v3, while v1 and v2 are maintained with security an v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning allocated sizes on allocation and free. -* 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory - was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose - for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows - fixed TLS offset, etc. -* 2025-03-28, `v1.9.3`, `v2.2.3`, `v3.0.3` (beta) : Various small bug and build fixes, including: - fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, - fix execution on non BMI1 x64 systems. -* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. - Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. - Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. - Upgrade mimalloc-redirect to v1.3.2. `MI_OPT_ARCH` is off by default now but still assumes armv8.1-a on arm64 - for fast atomic operations. Add QNX support. -* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS - guard pages behind objects to catch buffer overflows as they occur. - Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for - thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; - consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; - add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. * [Older release notes](#older-release-notes) @@ -369,14 +376,15 @@ Advanced options: - `MIMALLOC_PURGE_DECOMMITS=1`: By default "purging" memory means unused memory is decommitted (`MEM_DECOMMIT` on Windows, `MADV_DONTNEED` (which decresease rss immediately) on `mmap` systems). Set this to 0 to instead "reset" unused memory on a purge (`MEM_RESET` on Windows, generally `MADV_FREE` (which does not decrease rss immediately) on `mmap` systems). - Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual - address ranges and decommits within those ranges (to make the underlying physical memory available to other processes). + Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it will keep virtual + address ranges, but decommits within those ranges (to make the underlying physical memory available to other processes). Further options for large workloads and services: -- `MIMALLOC_ALLOW_THP=1`: By default always allow transparent huge pages (THP) on Linux systems. On Android only this is - by default off. When set to `0`, THP is disabled for the process that mimalloc runs in. If enabled, mimalloc also sets - the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when purging memory. +- `MIMALLOC_ALLOW_THP=[0,1,2]`: When set to `0`, the use of transparent huge pages (THP) is disabled for the process that mimalloc + runs in, and otherwise it is enabled whenever THP is [enabled in the OS][linux-thp] (`[always]` or `[madvise]`). If set to 2 (default on Linux), + mimalloc also sets the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when + purging memory which is better for performance (but may increase the rss!). On Android the default setting is 0 to reduce rss. - `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA @@ -403,6 +411,7 @@ Use caution when using `fork` in combination with either large or huge OS pages: for all pages in the original process including the huge OS pages. When any memory is now written in that area, the OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments. +[linux-thp]: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/performance_tuning_guide/sect-red_hat_enterprise_linux-performance_tuning_guide-configuring_transparent_huge_pages [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 @@ -419,7 +428,7 @@ to make mimalloc more robust against exploits. In particular: - The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc from the OS are also address randomized. -- If enabling `-DMI_SECURE_FULL=ON` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving +- If using `-DMI_SECURE=FULL` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive and can lead to reaching the maximum VMA limit on Linux systems if the heap gets too large. @@ -427,7 +436,7 @@ As always, evaluate with care as part of an overall security strategy as all of ## Debug Mode -When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`), +When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`) (or when using `-DMI_DEBUG=ON` explicitly), various checks are done at runtime to catch development errors. - Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. @@ -466,6 +475,33 @@ many system calls. Therefore, there are various environment variables (and optio Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. + +## Static override + +On Unix like systems, you can statically link with _mimalloc_ to override the standard +malloc interface. The recommended way is to link the final program with the +_mimalloc_ single object file (`mimalloc.o`). We use +an object file instead of a library file as linkers give preference to +that over archives to resolve symbols. To ensure that the standard +malloc interface resolves to the _mimalloc_ library, link it as the first +object file. For example: + +``` +> gcc -o myprogram mimalloc.o myfile1.c ... +``` + +Another way to override statically that works on all platforms, is to +link statically to mimalloc (as shown in the introduction) and include a +header file in each source file that re-defines `malloc` etc. to `mi_malloc`. +This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works +reliably though if all sources are +under your control or otherwise mixing of pointers from different heaps may occur! + +Note: recently we also enabled static overloading on Windows. In that case you need +to link with the static CRT _release_ runtime (`/MT`) and link with the static +`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). + + ## Dynamic override This is the recommended way to override the standard malloc interface. @@ -543,30 +579,55 @@ first in the import table. In such cases the [`minject`](bin) tool can be used to patch the executable's import tables. -## Static override +# Getting the best performance -On Unix-like systems, you can also statically link with _mimalloc_ to override the standard -malloc interface. The recommended way is to link the final program with the -_mimalloc_ single object file (`mimalloc.o`). We use -an object file instead of a library file as linkers give preference to -that over archives to resolve symbols. To ensure that the standard -malloc interface resolves to the _mimalloc_ library, link it as the first -object file. For example: +Consider the following build options (v3): -``` -> gcc -o myprogram mimalloc.o myfile1.c ... -``` +- `-DMI_OPT_ARCH=ON` (by default ON on arm64): enables architecture specific optimizations. + This uses `-march=haswell;-march=avx2` on x64 (available since 2013), `-march=armv8.1` on arm64 (2016), + and `-march=rv64gcb_zacas` on riscV (2023). + Each benefits mimalloc with better bit operations and/or native atomic operations (but also, if your + emulator or hardware does not support these extensions the program will fail with an illegal instruction). + On Apple arm64 the default is `-march=armv8.3` (which is supported on the M1 and better) which includes + efficient load-acquire instructions which mimalloc uses. For riscV the corresponding `zalasr` extension + can be enabled as `-DMI_OPT_ARCH=rv64gcb_zacas_zalasr` when your hardware supports it. -Another way to override statically that works on all platforms, is to -link statically to mimalloc (as shown in the introduction) and include a -header file in each source file that re-defines `malloc` etc. to `mi_malloc`. -This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works -reliably though if all sources are -under your control or otherwise mixing of pointers from different heaps may occur! +- `-DMI_ALLOW_THP=OFF`: disabling THP can reduce the rss of certain programs significantly. + +- `-DMI_ALLOW_THP=FULL`: enabling THP can increase performance of certain programs significantly. This setting + still disables THP if the OS THP setting is configured as `[never]`. + +- `-DMI_FREE_IS_CHECKED=ON` (by default ON in secure mode): slightly slower performance but checks validity + of every pointer passed to `mi_free`. Sometimes needed when overriding malloc/free process wide. + Note also that `mi_cfree` (checked free) always checks the validity of pointers passed to it which can + also be used if appropiate without needing to check all pointers process wide. + +In the source code: + +- Use `mi_zalloc` for zero initialized allocations. + +- Use the typed macros `mi_malloc_tp` etc. as these are both more safe and also use `mi_malloc_csize` internally. + +- Use `mi_malloc_csize`/`mi_free_csize` etc. when the allocated or free'd size is a constant as these are inlined + and statically redirect to `mi_malloc_small`/`mi_free_small` when possible. + These are also used by the C++ `mimalloc-new-delete.h` sized overrides. + You can use `mi_free_csize_nonnull` when the pointer is guaranteed to be not `NULL` as well. + +- Use `mi_malloc_small` when the allocation size is guaranteed to be less than `MI_SMALL_SIZE_MAX` (1KiB) for faster allocations. + +- Use `mi_free_small` when the pointer was returned from `mi_malloc_small` (and guaranteed to have a + size of less than `MI_SMALL_SIZE_MAX`). Use `mi_free_small_nonnull` when the pointer is guaranteed to be not `NULL` as well. + +For run-time systems and compilers (like Koka, Lean, etc.), we can do slightly better still. + +- If the allocation size is statically known, use `mi_malloc_csize` and `mi_free_csize` etc. when possible. + (or directly `mi_malloc_small`/`mi_free_small`). + +- If the runtime already carries thread local state, it may be faster to get the default `theap` for each thread + up-front (`mi_theap_get_default()`) and use the very fast `mi_theap_malloc(_small)` etc. passing the theap + pointer directly. This avoids having mimalloc look up the thread local pointer all the time. Whether this is + faster depends a bit on the OS implementation of thread local variables. -Note: recently we also enabled static overloading on Windows. In that case you need -to link with the static CRT _release_ runtime (`/MT`) and link with the static -`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). # Tools @@ -662,9 +723,11 @@ use a tool like [TraceControl] that is specialized for analyzing mimalloc traces [TraceControl]: https://github.com/xinglonghe/TraceControl + # Performance -Last update: 2021-01-30 +Last update: 2021-01-30. +(Note 2026-09-01: this is a while ago now, many of these allocators have improved over time as well. YMMV.) We tested _mimalloc_ against many other top allocators over a wide range of benchmarks, ranging from various real world programs to @@ -922,6 +985,25 @@ provided by the bot. You will only need to do this once across all repos using o # Older Release Notes +* 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory + was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose + for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows + fixed TLS offset, etc. +* 2025-03-28, `v1.9.3`, `v2.2.3`, `v3.0.3` (beta) : Various small bug and build fixes, including: + fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, + fix execution on non BMI1 x64 systems. +* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. + Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. + Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. + Upgrade mimalloc-redirect to v1.3.2. `MI_OPT_ARCH` is off by default now but still assumes armv8.1-a on arm64 + for fast atomic operations. Add QNX support. +* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS + guard pages behind objects to catch buffer overflows as they occur. + Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for + thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; + consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; + add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. + * 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches. * 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation. diff --git a/system/lib/mimalloc/src/alloc-aligned.c b/system/lib/mimalloc/src/alloc-aligned.c index 97dc6045b9973..111d63a8b2f9f 100644 --- a/system/lib/mimalloc/src/alloc-aligned.c +++ b/system/lib/mimalloc/src/alloc-aligned.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_theap_default +#include "mimalloc/prim-tls.h" // _mi_theap_default #include // memset @@ -18,7 +18,7 @@ terms of the MIT license. A copy of the license can be found in the file static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { // certain blocks are always allocated at a certain natural alignment. // (see also `arena.c:mi_arenas_page_alloc_fresh`). - mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0)); + mi_assert_internal(mi_alignment_is_valid(alignment)); if (alignment > size) return false; const size_t bsize = mi_good_size(size); const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N @@ -28,7 +28,7 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { } #if MI_GUARDED -static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept { +static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept { // use over allocation for guarded blocksl #if MI_THEAP_INITASNULL if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } @@ -39,7 +39,7 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, return NULL; } const size_t oversize = size + alignment - 1; - void* const base = _mi_theap_malloc_guarded(theap, oversize, zero); + void* const base = _mi_theap_malloc_guarded(theap, oversize, zero, ppage); if (base==NULL) return NULL; void* const p = _mi_align_up_ptr(base, alignment); mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size); @@ -48,31 +48,32 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, return p; } -static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) { +static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { #if MI_THEAP_INITASNULL if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } #endif const size_t rate = theap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` if (rate != 0) { theap->guarded_sample_rate = 0; } - void* p = _mi_theap_malloc_zero(theap, size, zero, usable); + void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); if (rate != 0) { theap->guarded_sample_rate = rate; } return p; } #else -static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) { - return _mi_theap_malloc_zero(theap, size, zero, usable); +static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { + return _mi_theap_malloc_zero(theap, size, zero, ppage); } #endif // Fallback aligned allocation that over-allocates -- split out for better codegen -static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)); - mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); + mi_assert_internal(mi_alignment_is_valid(alignment)); void* p; size_t oversize; + mi_page_t* page; if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { // use OS allocation for large alignments and allocate inside a singleton page (not in an arena) // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned @@ -84,7 +85,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block + p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { @@ -92,9 +93,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN); mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that. - p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable); + p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, &page); if (p == NULL) return NULL; } + mi_assert_internal(page == _mi_ptr_page(p)); + if (ppage!=NULL) { *ppage = page; } // .. and align within the allocation const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` @@ -107,9 +110,13 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // and we no longer own it. We should be careful to only read constant fields in the page, // or use safe atomic access as in `mi_page_set_has_interior_pointers`. // (we can access the page though since the just allocated pointer keeps it alive) - mi_page_t* page = _mi_ptr_page(p); if (aligned_p != p) { mi_page_set_has_interior_pointers(page, true); + // if (usable!=NULL) { + // mi_assert_internal(*usable > adjust); + // if (*usable > adjust) { *usable = *usable - adjust; } + // mi_assert_internal(*usable >= size); + // } #if MI_GUARDED // set tag to aligned so mi_usable_size works with guard pages if (adjust >= sizeof(mi_block_t)) { @@ -136,7 +143,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // // for the tracker, on huge aligned allocations only from the start of the large block is defined // mi_track_mem_undefined(aligned_p, size); // if (zero) { - // _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p)); + // _mi_memzero_(aligned_p, mi_usable_size(aligned_p)); // } //} @@ -150,12 +157,12 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } // Generic primitive aligned allocation -- split out for better codegen -static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { - mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); + mi_assert_internal(mi_alignment_is_valid(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { - _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); + _mi_error_message(EINVAL, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); return NULL; } @@ -163,7 +170,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable); + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, ppage); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { @@ -177,19 +184,21 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t } // fall back to over-allocation - return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable); + return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage); } +static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, size_t alignment, size_t offset) { + _mi_error_message(EINVAL, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); + return NULL; +} + // Primitive aligned allocation -static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept +static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. - if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see ) - #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment); - #endif - return NULL; + if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see ) + return mi_error_bad_alignment(size, alignment, offset); } #if MI_GUARDED @@ -197,7 +206,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con if mi_likely(theap!=NULL) #endif if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) { - return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero); + return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero, ppage); } #endif @@ -216,7 +225,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) { - if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if (ppage!=NULL) { *ppage = page; } void* p = _mi_page_malloc_zero(theap, page, padsize, zero); mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); @@ -228,7 +237,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con } // fallback to generic aligned allocation - return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable); + return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage); } @@ -248,7 +257,7 @@ static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL); } -static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { return mi_theap_zalloc_aligned_at(theap, size, alignment, 0); } @@ -275,8 +284,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment); } -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept { - return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size); +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); + if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + return p; } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { @@ -287,8 +299,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment); } -mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept { - return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size); +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); + if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + return p; } mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { @@ -330,24 +345,30 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, // ------------------------------------------------------ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { - mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL); + mi_assert(mi_alignment_is_valid(alignment)); + if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see ) + return mi_error_bad_alignment(newsize,alignment,offset); + } + if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero); if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL); - size_t size = mi_usable_size(p); - if (newsize <= size && newsize >= (size - (size / 2)) - && (((uintptr_t)p + offset) % alignment) == 0) { - return p; // reallocation still fits, is aligned and not more than 25% waste + const size_t size = mi_usable_size(p); + if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) { + return p; // reallocation still fits, is aligned and not more than 50% waste } else { - // note: we don't zero allocate upfront so we only zero initialize the expanded part - void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset); + // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size) + mi_page_t* newpage; + void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage); if (newp != NULL) { - if (zero && newsize > size) { + const size_t copy_size = (newsize > size ? size : newsize); + const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763 + if (zero && usable > zero_start) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized - size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); - _mi_memzero((uint8_t*)newp + start, newsize - start); + _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start); } - _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?) mi_free(p); // only free if successful } return newp; @@ -356,7 +377,7 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL); + if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero); return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero); } diff --git a/system/lib/mimalloc/src/alloc-override.c b/system/lib/mimalloc/src/alloc-override.c index 93d066dc1403a..04da5f2935d59 100644 --- a/system/lib/mimalloc/src/alloc-override.c +++ b/system/lib/mimalloc/src/alloc-override.c @@ -127,7 +127,7 @@ typedef void* mi_nothrow_t; #elif defined(_MSC_VER) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { - return mi_expand(_Block, _Size); + return mi__expand(_Block, _Size); } _Check_return_ _ACRTIMP size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT { @@ -324,14 +324,25 @@ typedef void* mi_nothrow_t; extern "C" { #endif +// defined here instead alloc-posix so we can alias it +mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + +mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + #ifndef MI_OSX_IS_INTERPOSED // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) - size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_size(const void* p) MI_FORWARD1(mi_malloc_size,p) #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) - size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_usable_size(void *p) MI_FORWARD1(mi_malloc_usable_size,p) #else - size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_malloc_usable_size,p) #endif // No forwarding here due to aliasing/name mangling issues diff --git a/system/lib/mimalloc/src/alloc-posix.c b/system/lib/mimalloc/src/alloc-posix.c index d4cf31dfe5c83..a2252494255af 100644 --- a/system/lib/mimalloc/src/alloc-posix.c +++ b/system/lib/mimalloc/src/alloc-posix.c @@ -31,34 +31,16 @@ terms of the MIT license. A copy of the license can be found in the file #define ENOMEM 12 #endif - -mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { - if (!mi_is_in_heap_region(p)) return 0; - return mi_usable_size(p); -} - -mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { - if (!mi_is_in_heap_region(p)) return 0; - return mi_usable_size(p); -} - mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { return mi_good_size(size); } -void mi_cfree(void* p) mi_attr_noexcept { - if (mi_is_in_heap_region(p)) { - mi_free(p); - } -} - int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noexcept (issue #794) // Note: The spec dictates we should not modify `*p` on an error. (issue#27) // if (p == NULL) return EINVAL; - if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment - // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned` - if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 + // it is required that alignment is a power of 2 and a multiple of sizeof(void*) + if (alignment // memset, strlen (for mi_strdup) #include // malloc, abort @@ -26,10 +26,37 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if MI_PADDING +static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept { + const size_t bsize = mi_page_usable_block_size(page); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); + ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); + #if (MI_DEBUG>=2) + mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); + #endif + mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + padding->canary = mi_ptr_encode_canary(page,block,page->keys); + padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK_BYTES + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)padding - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + } + #endif +} +#else +static mi_decl_maybe_unused mi_decl_noinline void* mi_block_zero(mi_block_t* block, size_t bsize) mi_attr_noexcept { + // const size_t bsize = mi_page_usable_block_size(page); + _mi_memzero_aligned(block, bsize); + return block; +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { if (page->block_size != 0) { // not the empty theap mi_assert_internal(mi_page_block_size(page) >= size); @@ -39,15 +66,24 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // check the free list mi_block_t* const block = page->free; - if mi_unlikely(block == NULL) { - return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable); + const mi_used_t used = page->used; + #if defined(__GNUC__) + __asm("" : : : "memory"); // always load the `used` field before the test + #endif + if (block == NULL) { + return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); - if (usable != NULL) { *usable = mi_page_usable_block_size(page); }; + if (ppage != NULL) { *ppage = page; }; // pop from the free list - page->free = mi_block_next(page, block); - page->used++; + mi_block_t* next = mi_block_next(page,block); + mi_track_mem_undefined(block,sizeof(*block)); + #if MI_SECURE + if (!zero) block->next = 0; // don't leak internal data + #endif + page->free = next; + page->used = used+1; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); @@ -63,6 +99,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ const size_t bsize = mi_page_usable_block_size(page); mi_track_mem_undefined(block, bsize); + // track per-block statistics #if (MI_STAT>0) if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { mi_theap_stat_increase(theap, malloc_normal, bsize); @@ -75,44 +112,28 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ } #endif + // in debug mode initialize with 0xD0 + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } + #endif + // zero the block? note: we need to zero the full block size (issue #63) - if mi_likely(!zero) { - // #if MI_SECURE - block->next = 0; // don't leak internal data - // #endif - #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } - #endif - } - else { + if mi_unlikely(zero) { if (!page->free_is_zero) { - _mi_memzero_aligned(block, bsize); + _mi_memzero_aligned(block,bsize); } else { - block->next = 0; - mi_track_mem_defined(block, bsize); - } + block->next = 0; + mi_track_mem_defined(block, bsize); + } } - + + // setup padding (must come after zero'ing) #if MI_PADDING // && !MI_TRACK_ENABLED - mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>=2) - mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); - #endif - mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess - padding->canary = mi_ptr_encode_canary(page,block,page->keys); - padding->delta = (uint32_t)(delta); - #if MI_PADDING_CHECK - if (!mi_page_is_huge(page)) { - uint8_t* fill = (uint8_t*)padding - delta; - const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes - for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } - } - #endif + mi_page_block_setup_padding(page,block,size); #endif - return block; + return block; } // extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. @@ -120,33 +141,28 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz return mi_page_malloc_zero(theap, page, size, zero, NULL); } -#if MI_GUARDED -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept; -#endif - // main allocation primitives for small and generic allocation // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG - const uintptr_t tid = _mi_thread_id(); - mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local #endif #if (MI_PADDING || MI_GUARDED) if mi_unlikely(size == 0) { size = sizeof(void*); } #endif #if MI_GUARDED if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) { - return _mi_theap_malloc_guarded(theap, size, zero); + return _mi_theap_malloc_guarded(theap, size, zero, ppage); } #endif // get page in constant time, and allocate from it mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); - void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable); + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -158,22 +174,22 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non } // internal generic allocation -static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept +static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { #if MI_GUARDED #if MI_THEAP_INITASNULL if (theap!=NULL) #endif if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) { - return _mi_theap_malloc_guarded(theap, size, zero); + return _mi_theap_malloc_guarded(theap, size, zero, ppage); } #endif #if !MI_THEAP_INITASNULL mi_assert(theap!=NULL); #endif - mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id()); // theaps are thread local + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local mi_assert((huge_alignment & 1)==0); - void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p, size, zero); #if MI_DEBUG>3 @@ -185,16 +201,16 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL if (theap!=NULL) { - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall + return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); #endif } @@ -213,18 +229,18 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, s } // The main internal allocation functions -static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept { +static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable); + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept { +extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) @@ -233,26 +249,26 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable); + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable); +extern inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); } // Main allocation functions mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(theap, size, false, NULL); + return _mi_theap_malloc_zero_ex(theap, size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_theap_malloc(_mi_theap_default(), size); + return mi_theap_malloc(_mi_theap_default(), size); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -302,27 +318,43 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t return mi_heap_zalloc(heap, total); } +static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { + mi_assert_internal(page == _mi_ptr_page(p)); + if (pblock_size!=NULL) { + if (p!=NULL) { *pblock_size = mi_page_block_size(page); } + } + return p; +} + // Return usable size -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, true, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept { - return mi_theap_umalloc(_mi_theap_default(), size, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* pblock_size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; - return mi_uzalloc(total, usable); + return mi_uzalloc(total, pblock_size); } // Uninitialized `calloc` @@ -351,14 +383,14 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { return NULL; #else if (p == NULL) return NULL; - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); - const size_t size = _mi_usable_size(p,page); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand"); + const size_t size = _mi_page_usable_size(page,p); if (newsize > size) return NULL; return p; // it fits #endif } -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept { +static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { // if p == NULL then behave as malloc. // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) @@ -367,12 +399,17 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool ze if (p==NULL) { page = NULL; size = 0; - if (usable_pre!=NULL) { *usable_pre = 0; } + if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } } - else { - page = mi_validate_ptr_page(p,"mi_realloc"); - size = _mi_usable_size(p,page); - if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); } + else { + page = mi_ptr_page_validate(p,"mi_realloc"); + if mi_unlikely(page==NULL) { // invalid pointer + if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } + if (pblock_size_post!=NULL) { *pblock_size_post = 0; } + return NULL; + } + size = _mi_page_usable_size(page,p); + if (pblock_size_pre!=NULL) { *pblock_size_pre = mi_page_block_size(page); } } // check if we can reuse the existing block if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) @@ -381,39 +418,52 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool ze if (theap!=NULL) #endif { - if (mi_page_heap(page)==_mi_theap_heap(theap)) { // and within the same heap + if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } - if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); } + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(page); } return p; // reallocation still fits and not more than 50% waste } } } - // otherwise allocate a fresh block - void* newp = mi_theap_umalloc(theap,newsize,usable_post); + // note: we don't zero allocate upfront so we only zero initialize the expanded part + mi_page_t* newpage; // use block_size for zero-ing, issue #763 + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,&newpage); if mi_likely(newp != NULL) { - if (zero && newsize > size) { - // also set last word in the previous allocation to zero to ensure any padding is zero-initialized - const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); - _mi_memzero((uint8_t*)newp + start, newsize - start); + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } + const size_t copy_size = (newsize > size ? size : newsize); + const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); + if (zero && usable > zero_start) { + _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start); } else if (newsize == 0) { ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) } if mi_likely(p != NULL) { - const size_t copysize = (newsize > size ? size : newsize); - mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking.. - _mi_memcpy(newp, p, copysize); - mi_free(p); // only free the original pointer if successful // todo: optimize since page is known? + mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking.. + _mi_memcpy_aligned(newp, p, copy_size); + mi_free(p); // only free the original pointer if successful } } return newp; } +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL); +} + mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL); + // optimize p==NULL + if (p==NULL) { + return mi_theap_malloc(theap,newsize); + } + else { + return _mi_theap_realloc_zero(theap, p, newsize, false); + } } static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { @@ -430,8 +480,14 @@ static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_at return newp; } -static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL); +mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { + // optimize p==NULL + if (p==NULL) { + return mi_theap_zalloc(theap,newsize); + } + else { + return _mi_theap_realloc_zero(theap, p, newsize, true); + } } static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { @@ -449,8 +505,8 @@ mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_ return mi_theap_reallocn(_mi_theap_default(),p,count,size); } -mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept { - return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post); +mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); } // Reallocate but free `p` on errors @@ -543,13 +599,30 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names + if (fname==NULL || *fname==0) { + errno = EINVAL; + return NULL; + } char buf[PATH_MAX]; DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); if (res == 0) { - errno = GetLastError(); return NULL; + DWORD err = GetLastError(); + switch (err) { + case ERROR_LOCK_VIOLATION: + case ERROR_SHARING_VIOLATION: + case ERROR_INVALID_ACCESS: errno = EACCES; break; + case ERROR_INVALID_HANDLE: + case ERROR_INVALID_FUNCTION: errno = EINVAL; break; + case ERROR_PATH_NOT_FOUND: errno = ENOTDIR; break; + case ERROR_FILE_NOT_FOUND: errno = ENOENT; break; + case ERROR_NOT_ENOUGH_MEMORY: errno = ENOMEM; break; + default: errno = EIO; + } + return NULL; } else if (res > PATH_MAX) { - errno = EINVAL; return NULL; + errno = ENAMETOOLONG; + return NULL; } else if (resolved_name != NULL) { return resolved_name; @@ -623,8 +696,11 @@ The standard requires calling into `get_new_handler` and throwing the bad_alloc exception on failure. If we compile with a C++ compiler we can implement this precisely. If we use a C compiler we cannot throw a `bad_alloc` exception -but we call `exit` instead (i.e. not returning). +but we call `abort` instead (i.e. not returning). +Also, the standard requires calling the new handler until +it returns false, but we limit the total calls. -------------------------------------------------------*/ +#define MI_TRY_NEW_MAX (4) #ifdef __cplusplus #include @@ -646,10 +722,19 @@ static bool mi_try_new_handler(bool nothrow) { #endif return false; } - else { + else if (!nothrow) { h(); return true; } + else { + try { + h(); + } + catch(...) { // swallow std::bad_alloc + return false; // stop trying + } + return true; + } } #else typedef void (*std_new_handler_t)(void); @@ -686,7 +771,8 @@ static bool mi_try_new_handler(bool nothrow) { static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) { void* p = NULL; - while(p == NULL && mi_try_new_handler(nothrow)) { + for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) { + if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once p = mi_theap_malloc(theap,size); } return p; @@ -700,7 +786,6 @@ static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); } - mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { void* p = mi_theap_malloc(theap,size); if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); @@ -717,7 +802,6 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size return p; } - mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { @@ -737,43 +821,52 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size); } - mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { void* p = mi_malloc(size); if mi_unlikely(p == NULL) return mi_try_new(size, true); return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { - void* p; - do { - p = mi_malloc_aligned(size, alignment); +static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) { + void* p = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) { + if (!mi_alignment_is_valid(alignment)) return NULL; + p = mi_malloc_aligned(size,alignment); } - while(p == NULL && mi_try_new_handler(false)); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { + void* p = mi_malloc_aligned(size, alignment); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,false); return p; } mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { - void* p; - do { - p = mi_malloc_aligned(size, alignment); - } - while(p == NULL && mi_try_new_handler(true)); + void* p = mi_malloc_aligned(size, alignment); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,true); return p; } +static mi_decl_noinline void* mi_try_new_realloc(void* p, size_t newsize) { + void* q = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(false); i++) { + if (newsize > MI_MAX_ALLOC_SIZE) return NULL; + q = mi_realloc(p,newsize); + } + return q; +} + mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { - void* q; - do { - q = mi_realloc(p, newsize); - } while (q == NULL && mi_try_new_handler(false)); + void* q = mi_realloc(p, newsize); + if (q == NULL) return mi_try_new_realloc(p,newsize); return q; } mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { - mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc + mi_try_new_handler(false); return NULL; } else { @@ -786,8 +879,8 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { // We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) // and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` -static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) { - // TODO: we can still make padding work by moving it out of the guard page area +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { + // todo: we can still make padding work by moving it out of the guard page area mi_page_t* const page = _mi_ptr_page(block); mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_GUARDED; @@ -824,13 +917,14 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) { offset = MI_PAGE_MAX_OVERALLOC_ALIGN; } uint8_t* const p = (uint8_t*)block + offset; - mi_assert_internal(p == guard_page - obj_size); + mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); + if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } mi_track_align(block, p, offset, obj_size); mi_track_mem_defined(block, sizeof(mi_block_t)); return p; } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { // allocate multiple of page size ending in a guard page // ensure minimal alignment requirement? @@ -842,16 +936,17 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); if (block==NULL) return NULL; - void* const p = mi_block_ptr_set_guarded(block, obj_size); - if (p == NULL) return p; - if (zero) { - _mi_memzero_aligned(p,obj_size); // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size) + size_t usable_size = 0; + void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); + if (p == NULL) return NULL; + if (zero) { + _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) } // stats - mi_track_malloc(p, obj_size, zero); + mi_track_malloc(p, usable_size, zero); if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); #if MI_STAT>1 @@ -880,6 +975,8 @@ void* _mi_externs[] = { (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, + (void*)&mi_theap_zalloc_small, + (void*)&mi_theap_calloc, (void*)&mi_malloc, (void*)&mi_malloc_small, (void*)&mi_zalloc, diff --git a/system/lib/mimalloc/src/arena-meta.c b/system/lib/mimalloc/src/arena-meta.c deleted file mode 100644 index cee38caa169e3..0000000000000 --- a/system/lib/mimalloc/src/arena-meta.c +++ /dev/null @@ -1,179 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2019-2024, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ - -/* ---------------------------------------------------------------------------- - We have a special "mini" allocator just for allocation of meta-data like - the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`). - - We reuse the bitmap of the arena's for allocation of 64b blocks inside - an arena slice (64KiB). - We always ensure that meta data is zero'd (we zero on `free`) ------------------------------------------------------------------------------*/ - -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "bitmap.h" - -/* ----------------------------------------------------------- - Meta data allocation ------------------------------------------------------------ */ - -#define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE -#define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN - -// large enough such that META_MAX_SIZE > 4k (even on 32-bit) -#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit -#define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE -#define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512 -#define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE) - -#if MI_META_MAX_SIZE <= 4096 -#error "max meta object size should be at least 4KiB" -#endif - -typedef struct mi_meta_page_s { - _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released) - mi_memid_t memid; // provenance of the meta-page memory itself - mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block. -} mi_meta_page_t; - -static mi_decl_cache_align _Atomic(mi_meta_page_t*) mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL); - - -#if MI_DEBUG > 1 -static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) { - mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size()); - if (block_idx != NULL) { - *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE; - } - return mpage; -} -#endif - -static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) { - return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next); -} - -static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) { - mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN)); - mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE); - void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE)); - mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL)); - return p; -} - -// allocate a fresh meta page and add it to the global list. -static mi_meta_page_t* mi_meta_page_zalloc(void) { - // allocate a fresh arena slice - // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...) - mi_memid_t memid; - uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0, - true /* commit*/, (MI_SECURE==0) /* allow large? */, - NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid); - if (base == NULL) return NULL; - mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN)); - if (!memid.initially_zero) { - _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE); - } - - // guard pages - #if MI_SECURE >= 1 - _mi_os_secure_guard_page_set_at(base, memid); - _mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid); - #endif - - // initialize the page and free block bitmap - mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size()); - mpage->memid = memid; - mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */); - const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL); - const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE); - const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE); - mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE); - mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks); - - // push atomically in front of the meta page list - // (note: there is no ABA issue since we never free meta-pages) - mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages); - do { - mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old); - } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage)); - return mpage; -} - - -// allocate meta-data -mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid ) -{ - mi_assert_internal(pmemid != NULL); - size = _mi_align_up(size,MI_META_BLOCK_SIZE); - if (size == 0 || size > MI_META_MAX_SIZE) return NULL; - const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE); - mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS); - mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages); - mi_meta_page_t* mpage = mpage0; - while (mpage != NULL) { - size_t block_idx; - if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) { - // found and claimed `block_count` blocks - *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count); - return mi_meta_block_start(mpage,block_idx); - } - else { - mpage = mi_meta_page_next(mpage); - } - } - // failed to find space in existing pages - if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) { - // the page list was updated by another thread in the meantime, retry - return _mi_meta_zalloc(size,pmemid); - } - // otherwise, allocate a fresh metapage and try once more - mpage = mi_meta_page_zalloc(); - if (mpage != NULL) { - size_t block_idx; - if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) { - // found and claimed `block_count` blocks - *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count); - return mi_meta_block_start(mpage,block_idx); - } - } - // if all this failed, allocate from the OS - return _mi_os_alloc(size, pmemid); -} - -// free meta-data -mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) { - if (p==NULL) return; - if (memid.memkind == MI_MEM_META) { - mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count); - const size_t block_count = memid.mem.meta.block_count; - const size_t block_idx = memid.mem.meta.block_index; - mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page; - mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage); - mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE); - mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count)); - // we zero on free (and on the initial page allocation) so we don't need a "dirty" map - _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE); - mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count); - } - else { - _mi_arenas_free(p,size,memid); - } -} - -// used for debug output -bool _mi_meta_is_meta_page(void* p) -{ - mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages); - mi_meta_page_t* mpage = mpage0; - while (mpage != NULL) { - if ((void*)mpage == p) return true; - mpage = mi_meta_page_next(mpage); - } - return false; -} diff --git a/system/lib/mimalloc/src/arena.c b/system/lib/mimalloc/src/arena.c index f90b75ea7220b..2bda823bc972f 100644 --- a/system/lib/mimalloc/src/arena.c +++ b/system/lib/mimalloc/src/arena.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2025, Microsoft Research, Daan Leijen +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -21,12 +21,9 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" #include "bitmap.h" -#if (MI_ARENA_MAX_SIZE >= MI_SIZE_SIZE*UINT32_MAX) -#error "The page_t.page_woffset field is not large enough to cover a full arena (redefine it to be an offset in MI_MAX_ALIGN_SIZE sizes?)" -#endif /* ----------------------------------------------------------- Arena id's @@ -90,36 +87,56 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page #endif size_t mi_arena_min_alignment(void) { - return MI_ARENA_SLICE_ALIGN; + return MI_ARENA_ALIGNMENT; } size_t mi_arena_min_size(void) { return MI_ARENA_MIN_SIZE; } -static size_t mi_arena_max_object_size(void) { +// slices reserved for page meta info at the start of aligned chunks +static size_t mi_arena_page_meta_aligned_slice_count(void) { + #if MI_PAGE_META_IS_ALIGNED + return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} + +// fixed limit for the maximum object size in an arena +static size_t mi_arena_max_fixed_object_size(void) { + #if MI_PAGE_META_IS_ALIGNED + return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); + #else + return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info + #endif +} + +// Maximum object size allowed to be allocated in an arena +size_t mi_arena_max_object_size(void) { size_t max_size = mi_option_get_size(mi_option_arena_max_object_size); max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE); if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { return MI_ARENA_MIN_OBJ_SIZE; } - else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) { // minus a bchunk to accommodate meta info - return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE); + else if (max_size >= mi_arena_max_fixed_object_size()) { + return mi_arena_max_fixed_object_size(); } else { return max_size; } } -mi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) { +mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(subproc!=NULL); if (arena != NULL && arena->commit_fun != NULL) { return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); } - else if (already_committed > 0) { - return _mi_os_commit_ex(start, size, is_zero, already_committed); + else if (stat_already_committed > 0) { + return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed); } else { - return _mi_os_commit(start, size, is_zero); + return _mi_os_commit(subproc, start, size, is_zero); } } @@ -137,7 +154,7 @@ static size_t mi_arena_size(mi_arena_t* arena) { // Start of the arena memory area static uint8_t* mi_arena_start(mi_arena_t* arena) { - return ((uint8_t*)arena); + return ((uint8_t*)arena->start); } // Start of a slice @@ -148,17 +165,22 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_assert_internal(slice_index < arena->slice_count); + #if MI_PAGE_META_IS_ALIGNED + mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize? + return page; + #else if (arena->pages_meta != NULL) { mi_page_t* const page = &arena->pages_meta[slice_index]; - #if MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_SMALL_IS_ALIGNED // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) if (page->block_size>0) return page; #else return page; #endif - } - // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL) + } + // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) return (mi_page_t*)mi_arena_slice_start(arena,slice_index); + #endif } // Arena area @@ -211,7 +233,6 @@ static size_t mi_page_full_size(mi_page_t* page) { } } - /* ----------------------------------------------------------- Arena Allocation ----------------------------------------------------------- */ @@ -219,6 +240,8 @@ static size_t mi_page_full_size(mi_page_t* page) { static mi_decl_noinline void* mi_arena_try_alloc_at( mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid) { + mi_assert_internal(arena!=NULL); + mi_assert_internal(slice_count>0); size_t slice_index; if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL; @@ -235,6 +258,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( mi_assert_internal(already_dirty <= touched_slices); touched_slices -= already_dirty; } + else { + // todo: properly count touched pages with a separate bitmap? + touched_slices = 0; + } // set commit state if (commit) { @@ -243,7 +270,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( if (already_committed < slice_count) { // not all committed, try to commit now bool commit_zero = false; - if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) { + if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) { // if the commit fails, release ownership, and return NULL; // note: this does not roll back dirty bits but that is ok. mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); @@ -268,7 +295,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( } else { // already fully committed. - _mi_os_reuse(p, mi_size_of_slices(slice_count)); + _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count)); // if the OS has overcommit, and this is the first time we access these pages, then // count the commit now (as at arena reserve we didn't count those commits as these are on-demand) if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) { @@ -516,7 +543,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc( if (req_arena != NULL) return NULL; // don't create arena's while preloading (todo: or should we?) - if (_mi_preloading()) return NULL; + // if (_mi_preloading()) return NULL; // don't create arena's if OS allocation is disallowed if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL; @@ -544,6 +571,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc( // Allocate from the OS (if allowed) static void* mi_arena_os_alloc_aligned( + mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid) @@ -555,10 +583,10 @@ static void* mi_arena_os_alloc_aligned( } if (align_offset > 0) { - return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid); + return _mi_os_alloc_aligned_at_offset(subproc, size, alignment, align_offset, commit, allow_large, memid); } else { - return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid); + return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid); } } @@ -583,7 +611,7 @@ void* _mi_arenas_alloc_aligned( mi_heap_t* heap, } // fall back to the OS - void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid); + void* p = mi_arena_os_alloc_aligned(heap->subproc, size, alignment, align_offset, commit, allow_large, req_arena, memid); return p; } @@ -600,10 +628,10 @@ void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_lar // release ownership of a page. This may free the page if all blocks were concurrently // freed in the meantime. Returns true if the page was freed. -static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) { +static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { @@ -611,8 +639,8 @@ static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) while mi_unlikely(mi_tf_block(tf_old) != NULL) { _mi_page_free_collect(page, false); // update used if (mi_page_all_free(page)) { // it may become free just before unowning it - _mi_arenas_page_unabandon(page, current_theap); - _mi_arenas_page_free(page, current_theap); + _mi_arenas_page_unabandon(page, current_theapx); + _mi_arenas_page_free(page, current_theapx); return true; } tf_old = mi_atomic_load_relaxed(&page->xthread_free); @@ -659,7 +687,8 @@ static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, siz mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count); mi_assert_internal(arena != NULL); if (parena_pages != NULL) { - mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(mi_page_heap(page), arena); + mi_heap_t* heap = mi_page_heap(page); + mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); mi_assert_internal(arena_pages != NULL); mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index)); *parena_pages = arena_pages; @@ -735,7 +764,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl _mi_page_free_collect(page, false); // update `used` count mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); @@ -749,8 +778,8 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl return NULL; } -static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { - MI_UNUSED_RELEASE(block_size); +static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { + MI_UNUSED(max_page_meta_count); mi_assert_internal(parena_pages!=NULL); *parena_pages = NULL; @@ -775,7 +804,7 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena); *parena_pages = arena_pages; if (arena_pages==NULL) { - _mi_arenas_free(start, mi_size_of_slices(slice_count), *memid); // roll back + _mi_arenas_free(heap->subproc, start, mi_size_of_slices(slice_count), *memid); // roll back start = NULL; } else { @@ -789,62 +818,146 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ // otherwise fall back to the OS if (start == NULL) { + #if MI_PAGE_META_IS_ALIGNED + size_t page_offset; // offset in the block for the page area + uint8_t* os_start; + if (block_alignment < MI_PAGE_META_ALIGNMENT) { + page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); + os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + else { + // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer + // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. + _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); + errno = EINVAL; + return NULL; + // page_offset = MI_PAGE_META_ALIGNMENT; + // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + if (os_start==NULL) return NULL; + // commit page info and page + const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; + const size_t min_page_meta = min_page_count * sizeof(mi_page_t); + mi_assert_internal(min_page_meta < page_offset); + bool is_zero; + bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); + if (ok && commit) { + ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); + } + if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } + if (!is_zero && !memid->initially_zero) { + _mi_memzero_aligned(os_start,min_page_meta); + } + start = os_start + page_offset; + mi_assert_internal(_mi_is_aligned(start,block_alignment)); + mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); + mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); + mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta); + memid->initially_committed = true; // so we don't commit again + #else if (os_align) { // note: slice_count already includes the page - mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment)); - start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); + start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); + mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment)); } else { - start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); + start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); } + #endif } if (start == NULL) return NULL; mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN)); - mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment)); return start; } -static size_t mi_page_block_start(size_t block_size, bool os_align) +// Only used for non-separate pages +mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool os_align) { + size_t offset; #if MI_GUARDED // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages) const size_t os_page_size = _mi_os_page_size(); mi_assert_internal(MI_PAGE_ALIGN >= os_page_size); if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) { - return _mi_align_up(mi_page_info_size(), os_page_size); + offset = _mi_align_up(mi_page_info_size(), os_page_size); } else #endif if (os_align) { - return MI_PAGE_ALIGN; + offset = MI_PAGE_ALIGN; } else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) - return _mi_align_up(mi_page_info_size(), block_size); + offset = _mi_align_up(mi_page_info_size(), block_size); + if (block_size < 64) { offset += 3*block_size; } } else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) - return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); + offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); } else { // otherwise start after the info - return mi_page_info_size(); + offset = mi_page_info_size(); } + return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); } + // Free a page without modifying page_bin stats static void mi_arenas_page_free_prim(mi_page_t* page); +static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { + #if MI_PAGE_META_IS_ALIGNED + if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) { + // ensure the meta data is committed + if (memid_slice.memkind == MI_MEM_ARENA) { + mi_arena_t* const arena = memid_slice.mem.arena.arena; + uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); + mi_assert_internal(meta_slices >= mi_arena_start(arena)); + const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; + if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { + // try to commit all page meta slices now + const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); + // the following assertion does not hold in a concurrent setting.. + // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); + const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; + if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) { + // if the commit fails return NULL + return NULL; + } + // set the commit bits + mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); + } + } + mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start); + return page_meta; + } + #else + if (memid_slice.memkind == MI_MEM_ARENA) { + MI_UNUSED(slice_start); + mi_arena_t* const arena = memid_slice.mem.arena.arena; + if (arena->pages_meta != NULL) { + mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); + return &arena->pages_meta[memid_slice.mem.arena.slice_index]; + } + } + #endif + return NULL; +} + // Allocate a fresh page static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) { const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); + const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); + const size_t alloc_size = mi_size_of_slices(slice_count); mi_memid_t memid = _mi_memid_none(); mi_arena_pages_t* arena_pages = NULL; - uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid,&arena_pages); + uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages); if (!slice_start) return NULL; // guard page at the end of mimalloc page? @@ -861,46 +974,60 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou size_t block_start = 0; // allocate page meta info at the arena start? - if (memid.memkind == MI_MEM_ARENA) { - mi_arena_t* const arena = memid.mem.arena.arena; - if (arena->pages_meta != NULL) { - mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); - mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index]; - mi_assert_internal(page_meta->block_size == 0); - #if MI_PAGE_META_ALIGNED_FREE_SMALL - // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, - // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (block_size > MI_SMALL_SIZE_MAX) + mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start); + if (page_meta!=NULL) { + mi_assert_internal(page_meta->block_size == 0); + #if MI_PAGE_META_SMALL_IS_ALIGNED + // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, + // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) + if (block_size <= MI_SMALL_SIZE_MAX) { + // put page info in front of the slice + page = (mi_page_t*)slice_start; + block_start = mi_page_block_start(block_size, os_align); + } + else + #endif + { + page_meta_is_separate = true; + page = page_meta; + block_start = 0; + #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) + #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ #endif + if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && + _mi_is_power_of_two(block_size)) { - page = page_meta; - page_meta_is_separate = true; - block_start = 0; - #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) - #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ - #endif - if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { - block_start += block_size; - } - mi_assert_internal(page->block_size == 0); - _mi_memzero_aligned(page, sizeof(*page)); + block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment + if (block_size < 64) { block_start += 3*block_size; } } + mi_assert_internal(page->block_size == 0); + _mi_memzero_aligned(page, sizeof(*page)); } } if (page == NULL) { + #if MI_PAGE_META_IS_ALIGNED + // can only happen on failing to commit the page meta info + _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid); + return NULL; + #else // put page meta info in front of the slice page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); + #endif + } + mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0); + if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + mi_assert_internal(block_start % block_size == 0); // natural alignment (see also alloc_aligned.c) } // commit first block? size_t commit_size = 0; if (!memid.initially_committed) { - commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE); + commit_size = _mi_align_up(block_start + block_size, mi_page_min_commit_size()); if (commit_size > page_noguard_size) { commit_size = page_noguard_size; } bool is_zero = false; - if mi_unlikely(!mi_arena_commit( mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) { - _mi_arenas_free(slice_start, alloc_size, memid); + if mi_unlikely(!mi_arena_commit( _mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) { + _mi_arenas_free(_mi_theap_subproc(theap), slice_start, alloc_size, memid); return NULL; } } @@ -914,7 +1041,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // set the guard page #if MI_SECURE>=5 if (memid.initially_committed) { - _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid); + _mi_os_secure_guard_page_set_at(_mi_theap_subproc(theap), slice_start + page_noguard_size, memid); } #endif @@ -940,22 +1067,46 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // initialize the page start uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); - const size_t offset = start - (uint8_t*)page; - mi_assert_internal((offset % MI_SIZE_SIZE) == 0 && (offset / MI_SIZE_SIZE) < UINT32_MAX); - page->page_woffset = (uint32_t)(offset / MI_SIZE_SIZE); - + page->page_offset = start - (uint8_t*)page; + // initialize page meta-data page->reserved = (uint16_t)reserved; page->block_size = block_size; page->memid = memid; page->free_is_zero = memid.initially_zero; - mi_assert_internal((commit && commit_size==0) || (!commit && commit_size < UINT32_MAX)); - page->slice_committed = (uint32_t)commit_size; - + mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size()))); + page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size()); + + page->heap = _mi_theap_heap(theap); + mi_page_set_theap(page,theap); + // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) + + #if MI_PAGE_META_IS_ALIGNED + mi_assert_internal(page_meta!=NULL); + mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page); + if (slice_count > 1) { + // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice + for(size_t i = 1; i < max_page_meta_count; i++) { + mi_assert_internal(page_meta[i].block_size == 0); + mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page); + } + } + #if MI_DEBUG>1 + mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page); + if (reserved>1) { + mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page); + } + #endif + #endif + mi_assert_internal(page->free==NULL); mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); mi_assert_internal(mi_page_slice_start(page) == slice_start); + mi_assert_internal(mi_page_size(page) <= page_noguard_size); + // now register in the arena_pages if (arena_pages!=NULL) { @@ -979,7 +1130,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_block_size(page) == block_size); - mi_assert_internal(mi_page_is_abandoned(page)); + // mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_owned(page)); return page; @@ -996,15 +1147,15 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // 2. find a free block, potentially allocating a new arena const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); - const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages - (slice_count >= mi_slice_count_of_size(UINT32_MAX)) || // always commit pages too large to hold a 32-bit slice_committed + const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages + (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); if (page == NULL) return NULL; mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); if (!_mi_page_init(theap, page)) { - mi_arenas_page_free_prim(page); + _mi_arenas_page_free(page,theap); return NULL; } @@ -1014,8 +1165,12 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // Allocate a page containing one block (very large, or with large alignment) static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { + #if MI_PAGE_META_IS_ALIGNED + const size_t info_size = 0; + #else const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); - const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + #endif #if MI_SECURE < 2 const size_t slice_count = mi_slice_count_of_size(info_size + block_size); #else @@ -1027,7 +1182,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block mi_assert(page->reserved == 1); if (!_mi_page_init(theap, page)) { - _mi_arenas_free( page, mi_page_full_size(page), page->memid); + _mi_arenas_page_free(page,theap); return NULL; } @@ -1083,7 +1238,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { mi_arena_pages_t* arena_pages = NULL; mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1)); // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may @@ -1099,7 +1254,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { // we must do this since we may later allocate large spans over this page and cannot have a guard page in between #if MI_SECURE >= 5 if (!page->memid.is_pinned) { - _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid); + _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid); } #endif @@ -1110,18 +1265,20 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { size_t slice_count; MI_UNUSED(slice_count); mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); mi_assert_internal(arena_pages!=NULL); + mi_assert_internal(arena->subproc == mi_page_subproc(page)); mi_bitmap_clear(arena_pages->pages, slice_index); - if (page->slice_committed > 0) { + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { // if committed on-demand, set the commit bits to account commit properly - mi_assert_internal(mi_page_full_size(page) >= page->slice_committed); - const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative + mi_assert_internal(mi_page_full_size(page) >= slice_committed); + const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices)); mi_assert_internal(slice_count >= total_slices); if (total_slices > 0) { mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL); } // any left over? - const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE; + const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE; if (extra > 0) { // pretend it was decommitted already mi_subproc_stat_decrease(arena->subproc, committed, extra); @@ -1132,7 +1289,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { } } if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking - _mi_arenas_free( mi_page_slice_start(page), mi_page_full_size(page), page->memid); + _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid); } void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { @@ -1142,17 +1299,11 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_all_free(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(page->next==NULL && page->prev==NULL); - mi_assert_internal(current_theapx == NULL || _mi_thread_id()==current_theapx->tld->thread_id); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); - if (current_theapx != NULL) { - mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1); - mi_theap_stat_decrease(current_theapx, pages, 1); - } - else { - mi_heap_t* const heap = mi_page_heap(page); - mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1); - mi_heap_stat_decrease(heap, pages, 1); - } + mi_heap_t* const heap = mi_page_heap(page); + mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); + mi_theapx_stat_decrease(heap, current_theapx, pages, 1); mi_arenas_page_free_prim(page); } @@ -1160,18 +1311,18 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { Arena abandon ----------------------------------------------------------- */ -void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { +void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); mi_assert_internal(page->next==NULL && page->prev == NULL); - mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); // add to abandoned? - mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap==_mi_theap_heap(current_theap)); + mi_heap_t* heap = mi_page_heap(page); if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { // make available for allocations size_t bin = _mi_bin(mi_page_block_size(page)); @@ -1184,15 +1335,15 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { mi_assert_internal(!mi_page_is_singleton(page)); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); mi_page_set_abandoned_mapped(page); const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); MI_UNUSED(was_clear); mi_assert_internal(was_clear); mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); - mi_theap_stat_increase(current_theap, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theap); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); return; } } @@ -1200,7 +1351,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { // page is full (or a singleton), or the page is OS/externally allocated // leave as is; it will be reclaimed when an object is free'd in the page // but for non-arena pages, add to the subproc list so these can be visited - if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) { + if (page->memid.memkind != MI_MEM_ARENA) { mi_lock(&heap->os_abandoned_pages_lock) { // push in front page->prev = NULL; @@ -1209,8 +1360,8 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { heap->os_abandoned_pages = page; } } - mi_theap_stat_increase(current_theap, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theap); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); } @@ -1229,16 +1380,13 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { } else { // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap - mi_theap_t* const theap = _mi_page_associated_theap_peek(page); - if (theap == NULL) { - return false; - } - else { - mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1); - mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1); // adjust as we are not abandoning fresh - _mi_arenas_page_abandon(page, theap); - return true; - } + mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL + mi_heap_t* const heap = mi_page_heap(page); + // if (theapx==NULL) return false; + mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1); + mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh + _mi_arenas_page_abandon(page, theapx); + return true; } } @@ -1248,7 +1396,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(current_theapx==NULL || _mi_thread_id()==current_theapx->tld->thread_id); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); mi_heap_t* const heap = mi_page_heap(page); if (mi_page_is_abandoned_mapped(page)) { @@ -1262,17 +1410,17 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); // this busy waits until a concurrent reader (from alloc_abandoned) is done - mi_bitmap_clear_once_set(arena_pages->pages_abandoned[bin], slice_index); + mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); mi_page_clear_abandoned_mapped(page); mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); } else { // page is full (or a singleton), page is OS allocated // if not an arena page, remove from the subproc os pages list - if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) { + if (page->memid.memkind != MI_MEM_ARENA) { mi_lock(&heap->os_abandoned_pages_lock) { if (page->prev != NULL) { page->prev->next = page->next; } if (page->next != NULL) { page->next->prev = page->prev; } @@ -1282,12 +1430,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { } } } - if (current_theapx!=NULL) { - mi_theap_stat_decrease(current_theapx, pages_abandoned, 1); - } - else { - mi_heap_stat_decrease(heap, pages_abandoned, 1); - } + mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); } @@ -1297,7 +1440,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices); static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); -void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) { +void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { if (p==NULL) return; if (size==0) return; @@ -1306,13 +1449,15 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) { if (mi_memkind_is_os(memid.memkind)) { // was a direct OS allocation, pass through - _mi_os_free(p, size, memid); + _mi_os_free(subproc, p, size, memid); } else if (memid.memkind == MI_MEM_ARENA) { // allocated in an arena size_t slice_count; size_t slice_index; mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count); + mi_assert_internal(arena!=NULL); + mi_assert_internal(arena->subproc == subproc); mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0); mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size); mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p); @@ -1342,8 +1487,8 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) { return; }; } - else if (memid.memkind == MI_MEM_META) { - _mi_meta_free(p, size, memid); + else if (memid.memkind == MI_MEM_MALLOC) { + _mi_free_subproc_safe(p); } else { // arena was none, external, or static; nothing to do @@ -1384,10 +1529,10 @@ static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) { return false; } -// Is a pointer inside any of our arenas? -bool _mi_arenas_contain(const void* p) { - return mi_arenas_contain_ex(p, NULL); -} +// // Is a pointer inside any of our arenas? +// bool _mi_arenas_contain(const void* p) { +// return mi_arenas_contain_ex(p, NULL); +// } // Is a pointer contained in the given arena area? bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) { @@ -1413,7 +1558,7 @@ static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { // mi_lock_done(&arena->abandoned_visit_lock); mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); if (mi_memkind_is_os(arena->memid.memkind)) { - _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the theap pointer (in `_mi_subproc()`) + _mi_os_free_ex(subproc, mi_arena_start(arena), mi_arena_size(arena), true, arena->memid); } } } @@ -1486,13 +1631,17 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { return size; } +static mi_arena_t* mi_arena_info(void* area) { + return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); +} + static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED const size_t pages_size = slice_count * sizeof(mi_page_t); #else const size_t pages_size = 0; @@ -1513,9 +1662,9 @@ static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) { return bitmap; } -static mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) { +static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) { mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base); - *base = (*base) + mi_bbitmap_init(bbitmap, slice_count, true /* already zero */); + *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */); return bbitmap; } @@ -1523,7 +1672,7 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { const size_t slice_count = arena->slice_count; size_t bitmap_base = 0; const size_t size = mi_arena_pages_size(slice_count, &bitmap_base); - mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(mi_heap_main(), size, MI_BCHUNK_SIZE); + mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE); if (arena_pages==NULL) return NULL; uint8_t* base = (uint8_t*)arena_pages + bitmap_base; mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE)); @@ -1539,7 +1688,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, int numa_node, bool exclusive, mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { - mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN)); + mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT)); mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now @@ -1558,9 +1707,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES); // return NULL; // } - - mi_arena_t* arena = (mi_arena_t*)start; - + // commit & zero if needed if (!memid.initially_committed) { size_t commit_size = mi_size_of_slices(info_slices); @@ -1568,10 +1715,10 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } bool ok = false; if (commit_fun != NULL) { - ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg); + ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); } else { - ok = _mi_os_commit(arena, commit_size, NULL); + ok = _mi_os_commit(subproc, start, commit_size, NULL); } if (!ok) { _mi_warning_message("unable to commit meta-data for OS memory"); @@ -1581,13 +1728,15 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, else if (!memid.is_pinned) { // if MI_SECURE, set a guard page at the end of the arena info // todo: this does not respect the commit_fun as the memid is of external memory - _mi_os_secure_guard_page_set_before((uint8_t*)arena + mi_size_of_slices(info_slices), memid); + _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid); } if (!memid.initially_zero) { - _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); + _mi_memzero(start, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); } // init + mi_arena_t* arena = mi_arena_info(start); + arena->start = start; arena->subproc = subproc; arena->memid = memid; arena->is_exclusive = exclusive; @@ -1607,7 +1756,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, // init bitmaps uint8_t* base = mi_arena_start(arena) + bitmap_base; - arena->slices_free = mi_arena_bbitmap_init(slice_count, &base); + arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base); arena->slices_committed = mi_arena_bitmap_init(slice_count, &base); arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base); arena->slices_purge = mi_arena_bitmap_init(slice_count, &base); @@ -1615,7 +1764,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); } - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED arena->pages_meta = (mi_page_t*)base; base += (slice_count * sizeof(mi_page_t)); #else @@ -1624,7 +1773,23 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); // reserve our meta info (and reserve slices outside the memory area) + #if MI_PAGE_META_IS_ALIGNED + for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { + // set all free slices (and skip the slices reserved for the page meta info) + const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count()); + const size_t start_idx = (i + meta_slices); + size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; + if (start_idx < arena->slice_count) { + if (count + start_idx > arena->slice_count) { + count = arena->slice_count - start_idx; + mi_assert_internal(count > 0); + } + mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); + } + } + #else mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); + #endif if (memid.initially_committed) { mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count); } @@ -1643,11 +1808,11 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s mi_assert(start!=NULL); if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } if (start==NULL) return false; - if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) { + if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) { // we can align the start since the memid tracks the real base of the memory. - void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE); + void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT); const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; - if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) { + if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) { _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); return false; } @@ -1730,12 +1895,18 @@ bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinne // Reserve a range of regular OS memory static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) { if (arena_id != NULL) *arena_id = _mi_arena_id_none(); - size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice + if (size <= MI_MAX_ALLOC_SIZE) { + size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice + } + if (size > MI_MAX_ALLOC_SIZE) { + _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size); + return ENOMEM; + } mi_memid_t memid; - void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid); + void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { - _mi_os_free_ex(start, size, commit, memid, NULL); + _mi_os_free_ex(subproc, start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); return ENOMEM; } @@ -1811,7 +1982,6 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { } static int mi_page_commit_usage(mi_page_t* page) { - // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100; const size_t committed_size = mi_page_committed(page); const size_t used_size = page->used * mi_page_block_size(page); return (int)(used_size * 100 / committed_size); @@ -1832,7 +2002,8 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, bit_set_count++; c = 'p'; color = MI_GRAY; - if (mi_page_is_singleton(page)) { c = 's'; } + if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; } + else if (mi_page_is_singleton(page)) { c = 's'; } else if (mi_page_is_full(page)) { c = 'f'; } if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); } int commit_usage = mi_page_commit_usage(page); @@ -1845,8 +2016,13 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, else { c = '?'; if (bit_of_page > 0) { c = '-'; } - else if (_mi_meta_is_meta_page(start)) { c = 'm'; color = MI_GRAY; } + // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } + #if MI_PAGE_META_IS_ALIGNED + else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) { + { c = 'i'; color = MI_GRAY; } + } + #endif // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; } @@ -1953,13 +2129,13 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro size_t page_total = 0; for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); - if (arena == NULL) break; + if (arena == NULL) continue; mi_assert(arena->subproc == subproc); // slice_total += arena->slice_count; - _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %p, numa: %i\n", + _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n", (arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB), (arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""), - arena->subproc, arena->numa_node); + arena->subproc->subproc_seq, arena->numa_node); //if (show_inuse) { // free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL); //} @@ -1974,7 +2150,7 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); // if (arena_pages != NULL) { - const char* header1 = "chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)"; + const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)"; const char* header2 = (narrow ? "\n " : " "); const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)"; page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count, @@ -2006,18 +2182,19 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m if (pages==0) return 0; if (numa_node < -1) numa_node = -1; if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); + mi_subproc_t* subproc = _mi_subproc(); size_t hsize = 0; size_t pages_reserved = 0; mi_memid_t memid; - void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid); + void* p = _mi_os_alloc_huge_os_pages(subproc, pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid); if (p==NULL || pages_reserved==0) { _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); return ENOMEM; } _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); - if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) { - _mi_os_free(p, hsize, memid); + if (!mi_manage_os_memory_ex2(subproc, p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) { + _mi_os_free(subproc, p, hsize, memid); return ENOMEM; } return 0; @@ -2097,7 +2274,7 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c size_t already_committed; mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..) const bool all_committed = (already_committed == slice_count); - const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg); + const bool needs_recommit = _mi_os_purge_ex(arena->subproc, p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg); if (needs_recommit) { // no longer committed @@ -2199,10 +2376,8 @@ static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) // expired yet? mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (!force) { - if (expire==0) return -1; - if (expire > now) return 0; - } + if (expire==0) return -1; + if (!force && expire > now) return 0; // reset expire mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0); @@ -2395,12 +2570,19 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are // move the page to `heap_target` as an abandoned page // first remove it from the current heap const size_t sbin = _mi_page_stats_bin(page); - size_t slice_index; - size_t slice_count; - mi_arena_pages_t* arena_pages = NULL; - mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); - mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index)); - mi_bitmap_clear(arena_pages->pages, slice_index); + mi_arena_t* arena = NULL; + size_t slice_index = 0; + if (page->memid.memkind == MI_MEM_ARENA) { + size_t slice_count; + mi_arena_pages_t* arena_pages = NULL; + arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); + mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index)); + mi_bitmap_clear(arena_pages->pages, slice_index); + } + else { + // os allocated + mi_assert_internal(mi_memid_is_os(page->memid) && page->next == NULL); + } if (theap != NULL) { mi_theap_stat_decrease(theap, page_bins[sbin], 1); mi_theap_stat_decrease(theap, pages, 1); @@ -2412,16 +2594,18 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are mi_theap_t* theap_target = info->theap_target; // and then add it to the new target heap - mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); - if mi_unlikely(arena_pages_target==NULL) { - // if we cannot allocate this, we move it to the main heap instead (which does not require allocation) - heap_target = mi_heap_main(); - theap_target = mi_heap_theap(heap_target); - arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); - mi_assert_internal(arena_pages_target!=NULL); - } - mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index)); - mi_bitmap_set(arena_pages_target->pages, slice_index); + if (arena != NULL) { + mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); + if mi_unlikely(arena_pages_target==NULL) { + // if we cannot allocate this, we move it to the main heap instead (which does not require allocation) + heap_target = mi_arena_heap_main(arena); + theap_target = mi_heap_theap(heap_target); // todo: find through theap_target tld? + arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena); + mi_assert_internal(arena_pages_target!=NULL); + } + mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index)); + mi_bitmap_set(arena_pages_target->pages, slice_index); + } page->heap = heap_target; mi_theap_stat_increase(theap_target, page_bins[sbin], 1); mi_theap_stat_increase(theap_target, pages, 1); @@ -2439,7 +2623,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) { _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info); #if MI_DEBUG>1 // no more arena pages? - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); if (arena_pages!=NULL) { mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages)); @@ -2459,7 +2643,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) { void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) { if (_mi_is_heap_main(heap_from)) return; - if (heap_to==NULL) { heap_to = mi_heap_main(); } + if (heap_to==NULL) { heap_to = heap_from->subproc->heap_main; } mi_heap_delete_pages(heap_from, heap_to); } diff --git a/system/lib/mimalloc/src/bitmap.c b/system/lib/mimalloc/src/bitmap.c index 1b9e2854fc812..01de1999a77dc 100644 --- a/system/lib/mimalloc/src/bitmap.c +++ b/system/lib/mimalloc/src/bitmap.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2024 Microsoft Research, Daan Leijen +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -107,9 +107,9 @@ static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bo } // Clear a bit but only when/once it is set. This is used by concurrent free's while -// the page is abandoned and mapped. This can incure a busy wait :-( but it should -// happen almost never (and is accounted for in the stats) -static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_t idx) { +// the page is abandoned and mapped. This can incur a busy wait :-( but it should +// be quite rare (and is accounted for in the stats) +static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomic(mi_bfield_t)*b, size_t idx) { mi_assert_internal(idx < MI_BFIELD_BITS); const mi_bfield_t mask = mi_bfield_mask(1, idx);; mi_bfield_t old = mi_atomic_load_relaxed(b); @@ -117,7 +117,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_ if mi_unlikely((old&mask) == 0) { old = mi_atomic_load_acquire(b); if ((old&mask)==0) { - mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1); + mi_subproc_stat_counter_increase(subproc, pages_unabandon_busy_wait, 1); } while ((old&mask)==0) { // busy wait _mi_prim_thread_yield(); @@ -133,8 +133,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_ // statistics correctly). static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) { mi_assert_internal(mask != 0); - mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {}; // try to atomically set the mask bits until success + const mi_bfield_t old = mi_atomic_or_acq_rel(b,mask); if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); } return ((old&mask) == 0); } @@ -143,8 +142,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t // `all_clear` is set to `true` if the new bfield became zero. static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) { mi_assert_internal(mask != 0); - mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {}; // try to atomically clear the mask bits until success + const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); if (all_clear != NULL) { *all_clear = ((old&~mask)==0); } return ((old&mask) == mask); } @@ -155,65 +153,79 @@ static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already return (old==0); } -// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) { -// const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero()); -// if (all_clear!=NULL) { *all_clear = true; } -// return (~old==0); -// } // ------- mi_bfield_atomic_try_clear --------------------------------------- - // Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0 -// and false otherwise (leaving the bit field as is). +// and false otherwise (leaving the bit field as is). Returns the acquired value back in `expect` (regardless of success). // `all_clear` is set to `true` if the new bfield became zero. -static inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) { +// `did_temp_clear_bits` is set to `true` some bits were temporarily cleared. +static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t* previous, bool* did_temp_clear_bits) { mi_assert_internal(mask != 0); + // note: we could also use a strong cas but generally an optimistic atomic and/or is more efficient (at least on arm64) // try to atomically clear the mask bits - do { - if ((expect & mask) != mask) { // are all bits still set? - if (all_clear != NULL) { *all_clear = (expect == 0); } - return false; + mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); + if (previous!=NULL) { *previous = old; } + if mi_likely((old&mask)==mask) { + // all bits in the mask transitioned from 1 to 0 atomically + return true; + } + else { + // failed to transition all + if ((old&mask)!=0) { + // restore accidentally cleared ones + mi_atomic_or_acq_rel(b, old&mask); + if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; } } - } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask)); - if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0); } - return true; -} - -static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) { - mi_assert_internal(mask != 0); - const mi_bfield_t expect = mi_atomic_load_relaxed(b); - return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear); + return false; + } } -// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0 -// and `false` otherwise leaving the bfield `b` as-is. +// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success. +// Returns `true` if the bit transitioned from 1 to 0 +// and `false` otherwise (leaving the bfield `b` as-is). // `all_clear` is set to true if the new bfield became zero (and false otherwise) -mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) { +mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear_optimistic(_Atomic(mi_bfield_t)* b, size_t idx, mi_bfield_t* previous) { mi_assert_internal(idx < MI_BFIELD_BITS); - const mi_bfield_t mask = mi_bfield_one()<bfields[field], mask, &already_set) : mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear)); - mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0)); + mi_assert_internal(!set || ((transition && already_set == 0) || (!transition && already_set > 0))); all_transition = all_transition && transition; total_already_set += already_set; maybe_all_clear = maybe_all_clear && all_clear; @@ -369,7 +381,7 @@ static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, b if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear); // if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear); // if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear); - // TODO: implement mi_bchunk_xsetNX instead of setNX + // todo: implement mi_bchunk_xsetNX instead of setNX return mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear); } @@ -445,14 +457,14 @@ static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, s // ------- mi_bchunk_try_clear --------------------------------------- // Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary. -static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) { +static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(cidx < MI_BCHUNK_BITS); mi_assert_internal(n <= MI_BFIELD_BITS); const size_t i = cidx / MI_BFIELD_BITS; const size_t idx = cidx % MI_BFIELD_BITS; if mi_likely(idx + n <= MI_BFIELD_BITS) { // within one field - return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear); + return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear, did_temp_clear_bits); } else { // spanning two fields (todo: use double-word atomic ops?) @@ -460,14 +472,15 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t mi_assert_internal(m < n); mi_assert_internal(i < MI_BCHUNK_FIELDS - 1); bool field1_is_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear, did_temp_clear_bits)) return false; // try the second field as well mi_assert_internal(n - m > 0); mi_assert_internal(n - m < MI_BFIELD_BITS); bool field2_is_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) { + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear, did_temp_clear_bits)) { // we failed to clear the second field, restore the first one mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL); + if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; } return false; } if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear; } @@ -488,7 +501,7 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t // and false otherwise leaving all bit fields as is. // Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway.. // `maybe_all_clear` is set to `true` if all the bfields involved become zero. -mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) { +mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); mi_assert_internal(n>0); if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; } @@ -504,7 +517,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci mi_assert_internal(start_field < MI_BCHUNK_FIELDS); const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx); bool maybe_all_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear, did_temp_clear_bits)) return false; // done? mi_assert_internal(m <= n); @@ -528,7 +541,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci mi_assert_internal(field < MI_BCHUNK_FIELDS); const mi_bfield_t mask_end = mi_bfield_mask(n, 0); bool field_is_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear, did_temp_clear_bits)) goto restore; maybe_all_clear = maybe_all_clear && field_is_clear; } @@ -538,6 +551,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci restore: // `field` is the index of the field that failed to set atomically; we need to restore all previous fields mi_assert_internal(field > start_field); + if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; } while( field > start_field) { field--; if (field == start_field) { @@ -551,11 +565,11 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci } -static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) { +static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(n>0); // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear); - if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear); - return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear); + if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); + return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); } @@ -576,19 +590,24 @@ mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) { } #endif + static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) { mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS); - // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever - // as the compiler won't reload the registers vec1 and vec2 from memory again. - const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]); - size_t idx; - if (mi_bfield_find_least_bit(b, &idx)) { // find the least bit - if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) { // clear it atomically - *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + mi_bfield_t b = mi_atomic_load_relaxed(bfield); + if (b==0) return false; + int tries = 0; + do { + const mi_bfield_t mask = (b & (~b+1)); // == (b & -b) but avoids a compiler warning -- clear all bits except the least-significant one + b = mi_atomic_and_acq_rel(bfield,~mask); // clear the bit and set `b` to the previous value + if mi_likely((b&mask)==mask) { // if we transitioned from 1 to 0, we actually cleared it + size_t bitidx = 0; + mi_bfield_find_least_bit(mask,&bitidx); + *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); return true; - } - } + } + } while (b!=0 && ++tries <= 4); // limit tries to reduce possible contention return false; } @@ -686,28 +705,36 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx return false; } -static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) { - mi_assert_internal(n==1); MI_UNUSED(n); +static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + mi_assert_internal(n==1); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); return mi_bchunk_try_find_and_clear(chunk, pidx); } -mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) { - const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]); - // has_set8 has low bit in each byte set if the byte in x == 0xFF - const mi_bfield_t has_set8 = - ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F - (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80 - >> 7; // shift high bit to low bit - size_t idx; - if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit - mi_assert_internal(idx <= (MI_BFIELD_BITS - 8)); - mi_assert_internal((idx%8)==0); - if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) { // unset the byte atomically - *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; - mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); - return true; +mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool* did_temp_clear_bits) { + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + mi_bfield_t b = mi_atomic_load_relaxed(bfield); + if (b==0) return false; + int tries = 0; + do { + // has_set8 has low bit in each byte set if the byte in x == 0xFF + const mi_bfield_t has_set8 = + ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F + (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80 + >> 7; // shift high bit to low bit + size_t bitidx; + if (mi_bfield_find_least_bit(has_set8, &bitidx)) { // find least 1-bit + mi_assert_internal(bitidx <= (MI_BFIELD_BITS - 8)); + mi_assert_internal((bitidx%8)==0); + if mi_likely(mi_bfield_atomic_try_clear8_optimistic(bfield,bitidx,&b,did_temp_clear_bits)) { // try to clear atomically (and update b) + *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; + mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); + return true; + } } - } + else { + return false; + } + } while(b!=0 && ++tries <= 4); // limit tries to reduce possible contention return false; } @@ -715,9 +742,9 @@ mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchu // set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success. // Used to find medium size pages in the free blocks. // todo: try neon version -static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) { +static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx, bool* did_temp_clear_bits) { #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) - while (true) { + for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries // since a cache-line is 64b, load all at once const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields); const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1); @@ -733,25 +760,29 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, s const size_t chunk_idx = bidx / 8; const size_t idx = (bidx % 8)*8; mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS); - if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) { // clear it atomically + if mi_likely(mi_bfield_atomic_try_clear8_optimistic(&chunk->bfields[chunk_idx], idx, NULL, did_temp_clear_bits)) { // clear it atomically *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); return true; } // try again - // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded } + // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded + // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206) + #if defined(__GNUC__) + __asm __volatile ("" : : "g"(chunk) : "memory"); + #endif } #else for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { - if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true; + if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx, did_temp_clear_bits)) return true; } - return false; #endif + return false; } -static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) { - mi_assert_internal(n==8); MI_UNUSED(n); - return mi_bchunk_try_find_and_clear8(chunk, pidx); +static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + mi_assert_internal(n==8); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); + return mi_bchunk_try_find_and_clear8(chunk, pidx, did_temp_clear_bits); } @@ -759,7 +790,7 @@ static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, // and try to clear them atomically. // set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. // will cross bfield boundaries. -mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) { +mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { if (n == 0 || n > MI_BFIELD_BITS) return false; const mi_bfield_t mask = mi_bfield_mask(n, 0); // for all fields in the chunk @@ -775,7 +806,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, const size_t bmask = mask<>idx == mask); if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically - if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) { + if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value *pidx = (i*MI_BFIELD_BITS) + idx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS); @@ -783,7 +814,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, } else { // if we failed to atomically commit, reload b and try again from the start - b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]); + b = b0; // = mi_atomic_load_acquire(&chunk->bfields[i]); } } else { @@ -803,7 +834,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, if (post + pre >= n) { // it fits -- try to claim it atomically const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post); - if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) { + if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL, did_temp_clear_bits)) { // we cleared all atomically *pidx = cidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); @@ -821,7 +852,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, // and try to clear them atomically. // set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. // This can cross bfield boundaries. -static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) { +static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically @@ -870,7 +901,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, // did we find a range? if (m==0) { - if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) { + if (mi_bchunk_try_clearN(chunk, cidx, n, NULL, did_temp_clear_bits)) { // we cleared all atomically *pidx = cidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); @@ -888,11 +919,11 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, // ------- mi_bchunk_clear_once_set --------------------------------------- -static inline void mi_bchunk_clear_once_set(mi_bchunk_t* chunk, size_t cidx) { +static inline void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx) { mi_assert_internal(cidx < MI_BCHUNK_BITS); const size_t i = cidx / MI_BFIELD_BITS; const size_t idx = cidx % MI_BFIELD_BITS; - mi_bfield_atomic_clear_once_set(&chunk->bfields[i], idx); + mi_bfield_atomic_clear_once_set(subproc, &chunk->bfields[i], idx); } @@ -963,7 +994,7 @@ static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) { return false; } -static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { +static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) { i--; mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); @@ -1323,9 +1354,10 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk return true; } else { - // failed to claim it, set abandoned mapping again (unless the page was freed) + // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false) if (keep_set) { const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL); + mi_bitmap_chunkmap_set(bitmap, chunk_idx); mi_assert_internal(wasclear); MI_UNUSED(wasclear); } } @@ -1391,12 +1423,12 @@ size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) { // Clear a bit once it is set. -void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx) { +void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx) { mi_assert_internal(idx < mi_bitmap_max_bits(bitmap)); const size_t chunk_idx = idx / MI_BCHUNK_BITS; const size_t cidx = idx % MI_BCHUNK_BITS; mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); - mi_bchunk_clear_once_set(&bitmap->chunks[chunk_idx], cidx); + mi_bchunk_clear_once_set(subproc, &bitmap->chunks[chunk_idx], cidx); } @@ -1459,6 +1491,8 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS); mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS); mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap)); + // clear rng bits in b + b = b & ~mi_bfield_mask(rng, bidx); if (!visit(idx, rng, arena, arg)) { // break early: reset the non-visited bits if (b!=0) { @@ -1466,8 +1500,6 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis } return false; } - // clear rng bits in b - b = b & ~mi_bfield_mask(rng, bidx); } mi_assert_internal(rngcount == bpopcount); } @@ -1505,14 +1537,15 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them - for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment + size_t shift; + for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift); if ((b & rngmask) == rngmask) { const size_t idx = base_idx + shift; if (!visit(idx, rngslices, arena, arg)) { // break early: restore non-visited entries mi_bfield_t notyet_visited = 0; - if (shift + rngslices < MI_BFIELD_BITS) { + if (rngslices + shift < MI_BFIELD_BITS) { notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices))); } mi_assert_internal((notyet_visited & skipped) == 0); @@ -1526,8 +1559,13 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo skipped = skipped | (b & rngmask); } } - + if (shift < MI_BFIELD_BITS) { + // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`. + mi_assert_internal(MI_BFIELD_BITS % rngslices != 0); + skipped = skipped | (b & (~(mi_bfield_t)0 << shift)); + } if (skipped != 0) { + // restore non-visited entries mi_atomic_or_relaxed(&chunk->bfields[j], skipped); } } @@ -1557,7 +1595,7 @@ size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) { // initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true // returns the size of the bitmap -size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) { +size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) { size_t chunk_count; const size_t size = mi_bbitmap_size(bit_count, &chunk_count); if (!already_zero) { @@ -1565,6 +1603,7 @@ size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zer } mi_atomic_store_release(&bbitmap->chunk_count, chunk_count); mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT); + bbitmap->subproc = subproc; return size; } @@ -1577,6 +1616,7 @@ void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) { // scan for highest zero bit in the bitmap // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck + // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding! const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap); for(size_t i = chunk_count; i > 0; ) { i--; @@ -1600,11 +1640,11 @@ static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) { if (ibin == bin) { const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); - if (was_clear) { mi_os_stat_increase(chunk_bins[ibin],1); } + if (was_clear) { mi_subproc_stat_increase(bbitmap->subproc, chunk_bins[ibin],1); } } else { const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL); - if (was_set) { mi_os_stat_decrease(chunk_bins[ibin],1); } + if (was_set) { mi_subproc_stat_decrease(bbitmap->subproc,chunk_bins[ibin],1); } } } } @@ -1701,8 +1741,15 @@ bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); if (cidx + n > MI_BCHUNK_BITS) return false; bool maybe_all_clear = false; - const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear); - if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); } + bool did_temp_clear_bits = false; + const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits); + if (cleared && maybe_all_clear) { + mi_assert_internal(!did_temp_clear_bits); + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } else if (did_temp_clear_bits) { + // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } // note: we don't set the size class for an explicit try_clearN (only used by purging) return cleared; } @@ -1744,14 +1791,14 @@ bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_ (used to find free pages) -------------------------------------------------------------------------------- */ -typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx); +typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits); // Go through the bbitmap and for every sequence of `n` set bits, call the visitor function. // If it returns `true` stop the search. // // This is used for finding free blocks and it is important to be efficient (with 2-level bitscan) // but also reduce fragmentation (through size bins). -static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) +static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) { // we space out threads to reduce contention const size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS); @@ -1805,7 +1852,8 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx]; size_t cidx; - if ((*on_find)(chunk, n, &cidx)) { + bool did_temp_clear_bits = false; + if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) { if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin // this chunk is now reserved for the `bbin` size class mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin); @@ -1815,9 +1863,17 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, return true; } else { + // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits) + // but that is ok as the chunkmap is a conservative approximation. // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?` - // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation. - mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + // probably not as we already only call `try_clear` once we fail to clear any bits. + if (did_temp_clear_bits) { + // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } + else { + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } } } mi_bfield_cycle_iterate_end(Y); @@ -1862,12 +1918,11 @@ bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t // Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) { mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap)); - size_t m = n; // bits to go size_t count = 0; // chunk count while (m > 0) { mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; - if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) { + if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) { goto rollback; } m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS); @@ -1881,6 +1936,8 @@ static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_ count--; mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL); + // since we may race with clearing, we need to set the chunkmap conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false); } return false; } @@ -1917,7 +1974,7 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t // did we find a suitable range? if (count == chunk_req) { - // now try to claim it! + // now try to claim it! if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) { *pidx = (chunk_idx * MI_BCHUNK_BITS); for (size_t i = 0; i < count; i++) { @@ -1926,6 +1983,11 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); return true; } + else { + // contended: we reset count to retry from the first + // (we still skip the first chunk to guarantee progress) + count = 0; + } } // keep searching but skip the scanned range diff --git a/system/lib/mimalloc/src/bitmap.h b/system/lib/mimalloc/src/bitmap.h index ca51a4e04736f..ba8224b8ab535 100644 --- a/system/lib/mimalloc/src/bitmap.h +++ b/system/lib/mimalloc/src/bitmap.h @@ -95,11 +95,11 @@ typedef mi_bchunk_t mi_bchunkmap_t; #define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS) #define MI_BITMAP_MIN_CHUNK_COUNT (1) -#if MI_SIZE_BITS > 32 -#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map -#else +// #if MI_SIZE_BITS > 32 +// #define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map +// #else #define MI_BITMAP_DEFAULT_CHUNK_COUNT (1) -#endif +// #endif #define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena #define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena #define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena @@ -199,7 +199,7 @@ mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t // Atomically clear a bit but only if it is set. Will block otherwise until the bit is set. // This is used to delay free-ing a page that it at the same time being considered to be // allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`). -void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx); +void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx); // If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit. @@ -260,8 +260,9 @@ static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) { typedef mi_decl_bchunk_align struct mi_bbitmap_s { _Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS) _Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set - #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2 - size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc by aligning manually + mi_subproc_t* subproc; // constant, for stats + #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 3 + size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 3]; // suppress warning on msvc by aligning manually #endif mi_bchunkmap_t chunkmap; mi_bchunkmap_t chunkmap_bins[MI_CBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE) @@ -288,7 +289,7 @@ bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx); // Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true // returns the size of the bitmap. -size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero); +size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero); // Set/clear a sequence of `n` bits in the bitmap (and can cross chunks). // Not atomic so only use if still local to a thread. diff --git a/system/lib/mimalloc/src/free.c b/system/lib/mimalloc/src/free.c index aa7366dcfe1e6..5d033aab44cb8 100644 --- a/system/lib/mimalloc/src/free.c +++ b/system/lib/mimalloc/src/free.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -9,12 +9,12 @@ terms of the MIT license. A copy of the license can be found in the file // add includes help an IDE #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_prim_thread_id() +#include "mimalloc/prim-tls.h" // _mi_prim_thread_id() #endif // forward declarations -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block); -static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); +mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); @@ -27,20 +27,26 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // fast path written carefully to prevent spilling on the stack static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full) { - MI_UNUSED(was_guarded); // checks - if mi_unlikely(mi_check_is_double_free(page, block)) return; - if (!was_guarded) { mi_check_padding(page, block); } - if (track_stats) { mi_stat_free(page, block); } + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; + if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding + + if (track_stats) { + mi_stat_free(page, block); + mi_track_free_size(block, usable_size); + } #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); + _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); #endif - if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned - + // actual free: push on the local free list + const mi_used_t used = page->used - 1; mi_block_set_next(page, block, page->local_free); + page->used = used; page->local_free = block; - if mi_unlikely(--page->used == 0) { + if mi_unlikely(used==0) { if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) _mi_page_retire(page); } @@ -54,35 +60,39 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept; // Free a block multi-threaded -static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept +static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept { - MI_UNUSED(was_guarded); - // adjust stats (after padding check and potentially recursive `mi_free` above) + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt + + // adjust stats (after padding check ) mi_stat_free(page, block); // stat_free may access the padding - mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); + mi_track_free_size(block, usable_size); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); -#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (!was_guarded) { - size_t dbgsize = mi_usable_size(block); - if (dbgsize > MI_MiB) { dbgsize = MI_MiB; } + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); } -#endif + #endif // push atomically on the page thread free list mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { mi_block_set_next(page, block, mi_tf_block(tf_old)); - tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */); + const bool new_owned = (allow_collect ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned + tf_new = mi_tf_create(block, new_owned); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough? // and atomically try to collect the page if it was abandoned - const bool is_owned_now = !mi_tf_is_owned(tf_old); - if (is_owned_now) { - mi_assert_internal(mi_page_is_abandoned(page)); - mi_free_try_collect_mt(page,block); + if (allow_collect) { + const bool is_owned_now = !mi_tf_is_owned(tf_old); + if (is_owned_now) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_free_try_collect_mt(page,block); + } } } @@ -96,10 +106,24 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { const size_t diff = (uint8_t*)p - mi_page_start(page); const size_t block_size = mi_page_block_size(page); - const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size); + size_t adjust = diff & (block_size - 1); + if mi_unlikely(!_mi_is_power_of_two(block_size)) { + adjust = diff % block_size; + } return (mi_block_t*)((uintptr_t)p - adjust); } +static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) { + mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer + #if MI_SECURE > 0 + // in secure mode we always unalign to guard against free-ing interior pointers + return _mi_page_ptr_unalign(page,p); + #else + MI_UNUSED(page); + return (mi_block_t*)p; + #endif +} + // forward declaration for a MI_GUARDED build #if MI_GUARDED static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration @@ -119,17 +143,6 @@ static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, vo } #endif -static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) { - mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer - #if MI_SECURE > 0 - // in secure mode we always unalign to guard against free-ing interior pointers - return _mi_page_ptr_unalign(page,p); - #else - MI_UNUSED(page); - return (mi_block_t*)p; - #endif -} - // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { @@ -140,49 +153,93 @@ static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_ } // free a pointer owned by another thread (page parameter comes first for better codegen) -static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept { +static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_collect) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_mt(page, block, was_guarded); + mi_free_block_mt(page, block, was_guarded, allow_collect); } // generic free (for runtime integration) void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept { if (is_local) mi_free_generic_local(page,p); - else mi_free_generic_mt(page,p); + else mi_free_generic_mt(page,p,true); } // Get the page belonging to a pointer // Does further checks in debug mode to see if this was a valid pointer. -static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) +static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const char* msg, bool free_small, bool check_p_for_null, mi_page_t** ppage) { - MI_UNUSED_RELEASE(msg); + MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); - return NULL; - } - mi_page_t* page = _mi_safe_ptr_page(p); - if (p != NULL && page == NULL) { - _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); + return false; } - return page; + #endif + + mi_page_t* page; + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); } + else + #endif + #if MI_PAGE_META_IS_ALIGNED + { page = _mi_aligned_ptr_page0(p); } #else - return _mi_ptr_page(p); + { page = _mi_ptr_page(p); } + #endif + + if mi_unlikely(check_p_for_null && page==NULL) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + return false; + } + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else + #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED + // for mi_free_small we can avoid a load-acquire (but not when guarded as that may still allocate large blocks) + if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else + #endif + { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } + #endif + + mi_assert_internal(page!=NULL); + mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); + #if !MI_GUARDED + if (free_small) { mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX) /* free small should only be called on small pages */); } #endif + *ppage = page; + return true; +} + +static mi_decl_forceinline bool mi_ptr_page_is_valid(const void* p, const char* msg, mi_page_t** ppage) { + return mi_ptr_page_is_valid_ex(p,msg,false /* free_small? */, true /* check_p_for_null */, ppage ); +} + +static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const char* msg) { + mi_page_t* page; + return (mi_ptr_page_is_valid(p,msg,&page) ? page : NULL); } // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page) +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) { - if mi_unlikely(page==NULL) return; // page will be NULL if p==NULL mi_assert_internal(p!=NULL && page!=NULL); - if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } - const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page)); + const mi_threadid_t ptid = mi_page_xthread_id(page); + const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); @@ -196,45 +253,117 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* p else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` // blocks are aligned (and not a full page); push on the thread_free list mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_mt(page,block,false /* was_guarded */); + mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); } else { // page is full or contains (inner) aligned blocks; use generic multi-thread path - mi_free_generic_mt(page, p); + mi_free_generic_mt(page, p, allow_collect); } } -void mi_free(void* p) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); - mi_free_ex(p, NULL, page); +void mi_free(void* p) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } } -void mi_ufree(void* p, size_t* usable) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree"); - mi_free_ex(p, usable, page); +void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { + mi_free_nonnull(p, page, pblock_size, true /* allow collect? */); + } + else { + if (pblock_size!=NULL) { *pblock_size = 0; } + } } void mi_free_small(void* p) mi_attr_noexcept { - // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. - // If we keep page info in front of the page area for small objects, we can find the info - // just by aligning down the pointer instead of looking it up in the page map. - #if MI_PAGE_META_ALIGNED_FREE_SMALL - #if MI_GUARDED - #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined" - mi_free(p); - #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE - #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } +} + +void mi_free_small_nonnull(void* p) mi_attr_noexcept { + mi_assert(p!=NULL); + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } +} + +// Free a pointer that is potentially allocated in a different sub-process +void _mi_free_subproc_safe(void* p) mi_attr_noexcept { + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { + mi_free_nonnull(p, page, NULL, false /* allow collect? */); + } +} + +// ------------------------------------------------------ +// Free variants +// ------------------------------------------------------ + +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + MI_UNUSED_RELEASE(size); + #if MI_DEBUG + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size"); + if (page==NULL) return; + mi_assert(p!=NULL); + const size_t usable = _mi_page_usable_size(page,p); + if mi_unlikely(size > usable) { + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + if (!is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } + } + if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + if (!is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } + } + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_free_small(p); + } + else + #endif + { mi_free(p); - #else - mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); - mi_assert(page == mi_validate_ptr_page(p,"mi_free_small")); - mi_assert((void*)page == _mi_align_down_ptr(mi_page_start(page),MI_SMALL_PAGE_SIZE)); - mi_assert(page->block_size <= MI_SMALL_SIZE_MAX); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` - mi_free_ex(p, NULL, page); - #endif - #else + } +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); mi_free(p); - #endif +} + +// checked free +bool mi_cfree(void* p) mi_attr_noexcept { + mi_page_t* const page = _mi_checked_ptr_page(p); + if mi_likely(page!=NULL) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + return true; + } + else { + return false; + } } @@ -315,7 +444,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long mi_assert_internal(!mi_page_all_free(page)); mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE); mi_assert_internal(reclaim_on_free >= 0); - + // dont reclaim if we just have terminated this thread and we should // not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944) if (!_mi_thread_is_initialized()) return false; @@ -359,6 +488,8 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); + // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses + // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list @@ -391,7 +522,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // ------------------------------------------------------ -// Usable size +// Usable size // ------------------------------------------------------ // Bytes available in a block @@ -399,16 +530,18 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p const mi_block_t* block = _mi_page_ptr_unalign(page, p); const bool is_guarded = mi_block_ptr_is_guarded(block,p); const size_t size = mi_page_usable_size_of(page, block, is_guarded); - const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block; - mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); - const size_t aligned_size = (size - adjust); + mi_assert_internal((void*)p >= (void*)block); + const size_t adjust = (uint8_t*)p - (uint8_t*)block; + mi_assert_internal(adjust <= size); + const size_t aligned_size = (adjust <= size ? size - adjust : 0); // size can be zero if the padding is corrupted return aligned_size; } -static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept { +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { if mi_unlikely(page==NULL) return 0; + mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); if mi_likely(!mi_page_has_interior_pointers(page)) { - const mi_block_t* block = (const mi_block_t*)p; + const mi_block_t* block = mi_validate_block_from_ptr(page,p); return mi_page_usable_size_of(page, block, false /* is guarded */); } else { @@ -418,85 +551,69 @@ static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_at } mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size"); - return _mi_usable_size(p,page); -} - - -// ------------------------------------------------------ -// Free variants -// ------------------------------------------------------ - -void mi_free_size(void* p, size_t size) mi_attr_noexcept { - MI_UNUSED_RELEASE(size); - #if MI_DEBUG - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); - const size_t available = _mi_usable_size(p,page); - mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ ); - #endif - mi_free(p); -} - -void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); - mi_free_size(p,size); -} - -void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); - mi_free(p); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_usable_size"); + return _mi_page_usable_size(page,p); } // ------------------------------------------------------ +// Deprecated: double free is usually checked with padding now +// as that is faster and works better for cross-thread free'ing. // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +#if MI_SECURE>=3 && !MI_PADDING // linear check if the free list contains a specific element -static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) { - while (list != NULL) { +static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { + const size_t max_count = page->capacity; // can never hold more blocks than the capacity + size_t count = 0; + while (list != NULL && count <= max_count) { // double-free can create cycles so we limit the number of iterations if (elem==list) return true; list = mi_block_next(page, list); + count++; + } + if mi_unlikely(count > max_count) { + _mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind); } return false; } -static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { +static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block) { // The decoded value is in the same page (or NULL). // Walk the free lists to verify positively if it is already freed - if (mi_list_contains(page, page->free, block) || - mi_list_contains(page, page->local_free, block) || - mi_list_contains(page, mi_page_thread_free(page), block)) + if (mi_list_contains(page, page->free, block, "free") || + mi_list_contains(page, page->local_free, block, "local free") || + mi_list_contains(page, mi_page_thread_free(page), block, "thread free")) { _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); - return true; + return false; } - return false; + return true; } -#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); } +// Used for double free checking to avoid checking free lists too frequently +static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { + mi_block_t* n = mi_block_nextx(page,block,page->keys); + return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? +} -static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { - bool is_double_free = false; - mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field - if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? - (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? +// check if `block` was free'd before +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { + if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL? { // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) - is_double_free = mi_check_is_double_freex(page, block); + return mi_check_double_freex(page, block); } - return is_double_free; + else return true; } #else -static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); - return false; + return true; } #endif @@ -506,16 +623,17 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block // --------------------------------------------------------------------------- #if MI_PADDING // && !MI_TRACK_ENABLED -static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { +static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) { *bsize = mi_page_usable_block_size(page); - const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); mi_track_mem_defined(padding,sizeof(mi_padding_t)); *delta = padding->delta; - uint32_t canary = padding->canary; - uintptr_t keys[2]; - keys[0] = page->keys[0]; - keys[1] = page->keys[1]; - bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize); + const uint32_t canary = padding->canary; + const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize); + if (double_free!=NULL) { + if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free? + else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed + } mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; } @@ -529,7 +647,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl else { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); mi_assert_internal(ok); mi_assert_internal(delta <= bsize); return (ok ? bsize - delta : 0); } @@ -542,7 +660,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); mi_assert_internal(ok); if (!ok || (bsize - delta) >= min_size) return; // usually already enough space mi_assert_internal(bsize >= min_size); @@ -571,16 +689,17 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si } #endif -#if MI_PADDING && MI_PADDING_CHECK +#if MI_PADDING -static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) { +static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free ); *size = *wrong = bsize; if (!ok) return false; mi_assert_internal(bsize >= delta); *size = bsize - delta; + #if MI_PADDING_CHECK_BYTES if (!mi_page_is_huge(page)) { uint8_t* fill = (uint8_t*)block + bsize - delta; const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes @@ -594,22 +713,37 @@ static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, si } mi_track_mem_noaccess(fill, maxpad); } + #endif return ok; } -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - size_t size; - size_t wrong; - if (!mi_verify_padding(page,block,&size,&wrong)) { - _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong ); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + if mi_unlikely(is_guarded) { + const size_t bsize = mi_page_block_size(page); + *usable_size = (bsize - _mi_os_page_size()); + return true; + } + else { + size_t wrong; + bool is_double_free; + if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) { + if (is_double_free) { + _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size); + } + else { + _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong ); + } + return false; + } + return true; } } #else -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(page); - MI_UNUSED(block); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + *usable_size = mi_page_usable_size_of(page,block,is_guarded); + return true; } #endif @@ -617,9 +751,18 @@ static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { // only maintain stats for smaller objects if requested #if (MI_STAT>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(block); - mi_theap_t* const theap = _mi_theap_default(); - if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS) + MI_UNUSED(block); + mi_theap_t* theap = _mi_theap_default(); + mi_lock_t* lock = NULL; + mi_subproc_t* const subproc = mi_page_subproc(page); + mi_theap_t* const theap_meta = subproc->theap_meta; + if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS) + // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page + (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) { + theap = theap_meta; + lock = &subproc->theap_meta_lock; + mi_lock_acquire(lock); + } const size_t bsize = mi_page_usable_block_size(page); // #if (MI_STAT>1) @@ -636,6 +779,10 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc mi_theap_stat_decrease(theap, malloc_huge, bpsize); } + + if mi_unlikely(lock!=NULL) { + mi_lock_release(lock); + } } #else void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { diff --git a/system/lib/mimalloc/src/heap.c b/system/lib/mimalloc/src/heap.c index f0a016527a6e5..3e7611b5ec84b 100644 --- a/system/lib/mimalloc/src/heap.c +++ b/system/lib/mimalloc/src/heap.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_theap_default +#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "mimalloc/prim-tls.h" // _mi_heap_theap /* ----------------------------------------------------------- @@ -15,7 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file ----------------------------------------------------------- */ mi_theap_t* mi_heap_theap(mi_heap_t* heap) { - return _mi_heap_theap(heap); + return _mi_heap_theap(heap); // in prim.h } void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) { @@ -30,64 +31,66 @@ void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) { void mi_heap_stats_merge_to_main(mi_heap_t* heap) { if (heap==NULL) return; - _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats); + _mi_stats_merge_into(&mi_heap_get_heap_main(heap)->stats, &heap->stats); } +bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) { + mi_assert_internal((uintptr_t)theap == 1 || _mi_theap_heap(theap)==heap); + mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(heap->theap != 0); + return _mi_thread_local_set(heap->theap,theap); +} + +// mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) { +// mi_theap_t* theap; +// mi_assert_internal(heap->theap != 0); +// if mi_likely(heap->theap!=0) { // paranoia +// theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); +// } +// else { +// _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap); +// return NULL; +// } +// mi_assert_internal(!_mi_is_empty_theap(theap)); +// mi_assert_internal(theap->heap == heap); // this goes wrong if using main heaps across subprocesses (as all share the same key) +// return theap; +// } + + static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap) { mi_heap_t* heap = (mi_heap_t*)const_heap; mi_assert_internal(heap!=NULL); - if (_mi_is_heap_main(heap)) { - // this can be called if the (main) thread is not yet initialized (as no allocation happened) - // but `theap_main_init_get()` will call `mi_thread_init()` - mi_theap_t* const theap = _mi_theap_main_safe(); - mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap))); - return theap; + // initialize thread first in case this is the main heap + // (which may allocate the default theap already for the main heap) + if (!_mi_thread_is_initialized()) { + mi_thread_init(); } - // otherwise initialize the theap for this heap - // get the thread local - mi_assert_internal(heap->theap != 0); - if (heap->theap==0) { // paranoia - _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap); - return NULL; - } + // get the thread local theap mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); // create a fresh theap? if (theap==NULL) { - // set first an invalid value to ensure the thread local storage is allocated - if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) { - _mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n"); - return NULL; - } - // then allocate the theap - theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld); - _mi_thread_local_set(heap->theap, theap); // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1) + // allocate a fresh theap + theap = _mi_theap_create(heap, mi_theap_get_default()->tld); // sets the theap thread local if (theap==NULL) { _mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n"); return NULL; - } + } + _mi_heap_theap_set(heap, theap); + mi_assert_internal(theap == (mi_theap_t*)_mi_thread_local_get(heap->theap)); } return theap; } -// get the theap for a heap without initializing (and return NULL in that case) -mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) { - if (heap==NULL || _mi_is_heap_main(heap)) { - return _mi_theap_main_safe(); - } - else { - return (mi_theap_t*)_mi_thread_local_get(heap->theap); - } -} - // get (and possibly create) the theap belonging to a heap mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) { - mi_theap_t* theap = _mi_heap_theap_peek(heap); + mi_assert_internal(heap->theap != 0); + mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if mi_unlikely(theap==NULL) { theap = mi_heap_init_theap(heap); if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic @@ -96,26 +99,12 @@ mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) return theap; } - -mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) { - // always allocate heap data in the (subprocess) main heap - mi_heap_t* const heap_main = mi_heap_main(); - // todo: allocate heap data in the exclusive arena ? - mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) ); - if (heap==NULL) return NULL; - - // reserve a thread local slot for this heap (see also issue #1230) - const mi_thread_local_t theap_slot = _mi_thread_local_create(); - if (theap_slot == 0) { - _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n"); - mi_free(heap); - return NULL; - } - +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id) +{ // init fields heap->theap = theap_slot; - heap->subproc = heap_main->subproc; - heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count); + heap->subproc = subproc; + heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count); heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id); heap->numa_node = -1; // no initial affinity mi_stats_header_init(&heap->stats); @@ -131,94 +120,135 @@ mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) { if (head!=NULL) { head->prev = heap; } heap->subproc->heaps = heap; } - mi_atomic_increment_relaxed(&heap_main->subproc->heap_count); - mi_subproc_stat_increase(heap_main->subproc, heaps, 1); + mi_atomic_increment_relaxed(&subproc->heap_count); + mi_subproc_stat_increase(subproc, heaps, 1); + mi_assert_internal(_mi_is_heap_main(heap) ? heap->theap == mi_thread_local_key_fast : heap->theap != 0); +} + +mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap) { + mi_assert_internal(is_main_heap ? (subproc->heap_main == NULL && subproc->parent != NULL) : subproc->heap_main != NULL); + // heap data is allocated in the current subproc + mi_heap_t* const heap_main = (is_main_heap ? subproc->parent->heap_main : subproc->heap_main); + // todo: allocate heap data in the exclusive arena ? + mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) ); + if (heap==NULL) return NULL; + // reserve a thread local slot for this heap (see also issue #1230) + mi_thread_local_t theap_slot = (is_main_heap ? mi_thread_local_key_fast : _mi_thread_local_create()); + if (theap_slot == 0) { + _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n"); + mi_free(heap); + return NULL; + } + if (is_main_heap) { + mi_assert_internal(subproc->heap_main == NULL); + subproc->heap_main = heap; + } + _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id); return heap; } +mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) { + // `mi_heap_new` may be the very first mimalloc call in a process, in which case the + // main heap does not exist yet and `_mi_heap_new_for_subproc` would allocate from a NULL `subproc->heap_main`. + mi_thread_init(); + return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false); +} + mi_heap_t* mi_heap_new(void) { return mi_heap_new_in_arena(0); } // free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency) static void mi_heap_free_theaps(mi_heap_t* heap) { - // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), + // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), // and we need to ensure we free theaps atomically. - // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock - // potential concurrent thread termination which tries to remove the theap from our theaps list. - bool all_freed; - do { - all_freed = true; - mi_theap_t* theap = NULL; - mi_lock(&heap->theaps_lock) { - theap = heap->theaps; - while(theap != NULL) { - mi_theap_t* next = theap->hnext; - if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) { - all_freed = false; - } - theap = next; - } - } - if (!all_freed) { - mi_heap_stat_counter_increase(heap,heaps_delete_wait,1); - _mi_prim_thread_yield(); + + // We first detach our theaps list from any thread local lists + _mi_heap_detach_theaps(heap); + + // Now we can safely free the theaps + mi_lock(&heap->theaps_lock) { // paranoia + mi_theap_t* theap = heap->theaps; + heap->theaps = NULL; + while(theap != NULL) { + mi_theap_t* next = theap->hnext; + theap->hnext = NULL; + theap->hprev = NULL; + mi_assert_internal(theap->tld==NULL); + // merge stats into the owning heap stats + _mi_stats_merge_into(&heap->stats, &theap->stats); + // and free + _mi_theap_decref(theap); // a cached entry can still point to the theap + theap = next; } - else { - mi_assert_internal(heap->theaps==NULL); - } - } - while(!all_freed); + } } // free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed) -static void mi_heap_free(mi_heap_t* heap) { - mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap)); +static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { + mi_assert_internal(heap!=NULL); // && !_mi_is_process_heap_main(heap)); // free all arena pages infos - mi_lock(&heap->arena_pages_lock) { - for (size_t i = 0; i < MI_MAX_ARENAS; i++) { - mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); - if (arena_pages!=NULL) { - mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); - mi_free(arena_pages); + const bool is_main = _mi_is_heap_main(heap); + if (!is_main) { // pages for the main heap are pre-allocated in the arenas + mi_lock(&heap->arena_pages_lock) { + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { + mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); + if (arena_pages!=NULL) { + mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); + _mi_free_subproc_safe(arena_pages); + } } } } // remove the heap from the subproc - mi_heap_stats_merge_to_main(heap); + if (!is_main) { + mi_heap_stats_merge_to_main(heap); + } + else { + _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); + } mi_atomic_decrement_relaxed(&heap->subproc->heap_count); mi_subproc_stat_decrease(heap->subproc, heaps, 1); - mi_lock(&heap->subproc->heaps_lock) { + mi_lock_maybe(&heap->subproc->heaps_lock, acquire_heaps_lock) { if (heap->next!=NULL) { heap->next->prev = heap->prev; } if (heap->prev!=NULL) { heap->prev->next = heap->next; } else { heap->subproc->heaps = heap->next; } } - _mi_thread_local_free(heap->theap); mi_lock_done(&heap->theaps_lock); mi_lock_done(&heap->os_abandoned_pages_lock); mi_lock_done(&heap->arena_pages_lock); - mi_free(heap); + if (!_mi_is_process_heap_main(heap)) { + _mi_thread_local_free(heap->theap); + _mi_free_subproc_safe(heap); + } } void mi_heap_delete(mi_heap_t* heap) { if (heap==NULL) return; - if (_mi_is_heap_main(heap)) { + mi_heap_t* heap_main = mi_heap_get_heap_main(heap); + if (heap == heap_main) { _mi_warning_message("cannot delete the main heap\n"); return; } mi_heap_free_theaps(heap); - _mi_heap_move_pages(heap, mi_heap_main()); - mi_heap_free(heap); + _mi_heap_move_pages(heap, heap_main); + mi_heap_free(heap,true /* acquire subproc->heaps_lock */); } -void _mi_heap_force_destroy(mi_heap_t* heap) { +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock) { if (heap==NULL) return; mi_heap_free_theaps(heap); _mi_heap_destroy_pages(heap); - if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap? + // if (_mi_subproc_main()->heap_main == heap) { + // _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); + // } + // else + { + mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap? + } } void mi_heap_destroy(mi_heap_t* heap) { @@ -227,17 +257,18 @@ void mi_heap_destroy(mi_heap_t* heap) { _mi_warning_message("cannot destroy the main heap\n"); return; } - _mi_heap_force_destroy(heap); + _mi_heap_force_destroy(heap,true /* acquire subproc->heaps_lock */); } mi_heap_t* mi_heap_of(const void* p) { - mi_page_t* page = _mi_safe_ptr_page(p); + mi_page_t* const page = _mi_safe_ptr_page(p); if (page==NULL) return NULL; return mi_page_heap(page); } bool mi_any_heap_contains(const void* p) { - return (mi_heap_of(p)!=NULL); + mi_page_t* const page = _mi_safe_ptr_page(p); + return (page!=NULL); } bool mi_heap_contains(const mi_heap_t* heap, const void* p) { @@ -257,7 +288,7 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc if (p==NULL) return false; const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false; - // If the page is the head of the queue, it is currently being used for + // If the page is the head of the queue, it is currently being used for // allocations; we skip it to avoid immediate thrashing. if (page->prev == NULL) return false; @@ -265,7 +296,7 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc const mi_heap_t* const page_heap = mi_page_heap(page); if (page_heap==NULL) return false; if (heap!=NULL && page_heap!=heap) return false; - + // check utilization if (page->capacity==0) return false; if (perc_threshold>=100) return true; diff --git a/system/lib/mimalloc/src/init.c b/system/lib/mimalloc/src/init.c index 39f4d03eb9d69..68e94aaa20f3d 100644 --- a/system/lib/mimalloc/src/init.c +++ b/system/lib/mimalloc/src/init.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -7,44 +7,59 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" #include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" #include // memcpy, memset #include // atexit -#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ } -#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC) - // Empty page used to initialize the small free pages array -const mi_page_t _mi_page_empty = { +static const mi_page_t mi_page_empty = { + #if MI_PAGE_META_IS_ALIGNED + MI_ATOMIC_VAR_INIT(NULL), // self + #endif MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free 0, // used + NULL, // local_free + 0, // block_size + 0, // page_offset 0, // capacity 0, // reserved capacity + 0, // slice_pcommitted 0, // retire_expire false, // is_zero - NULL, // local_free MI_ATOMIC_VAR_INIT(0), // xthread_free - 0, // block_size - 0, // page_woffset - MI_ARENA_SLICE_SIZE, // page_committed - #if (MI_PADDING || MI_ENCODE_FREELIST) - { 0, 0 }, // keys - #endif NULL, // theap NULL, // heap NULL, NULL, // next, prev - MI_MEMID_STATIC // memid + MI_MEMID_STATIC, // memid + #if (MI_PADDING || MI_ENCODE_FREELIST) + #if MI_PAGE_KEY_COUNT==2 + { 0, 0 }, // keys + #else + { 0 }, // key + #endif + // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + // { 0 }, // padding + #endif }; -#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) +#define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) + +#if MI_SMALL_WSIZE_MAX == 128 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p) +#elif MI_SMALL_WSIZE_MAX == 256 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p), MI_INIT128(p) +#else +#error define initializer for direct pages +#endif #if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #elif (MI_PADDING>0) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #else -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #endif @@ -70,15 +85,16 @@ const mi_page_t _mi_page_empty = { #define MI_STAT_COUNT(stat) {0,0,0}, #define MI_STAT_COUNTER(stat) {0}, -#define MI_STATS_NULL \ - MI_STAT_FIELDS() \ - \ - { MI_INIT4(MI_STAT_COUNT_NULL) }, \ - { { 0 }, { 0 }, { 0 }, { 0 } }, \ - \ - { MI_INIT74(MI_STAT_COUNT_NULL) }, \ - { MI_INIT74(MI_STAT_COUNT_NULL) }, \ - { MI_INIT5(MI_STAT_COUNT_NULL) } +#define MI_STATS_FIELDS_NULL \ + MI_STAT_FIELDS() /* regular stat fields */ \ + { MI_INIT4(MI_STAT_COUNT_NULL) }, /* stat reserved */ \ + { { 0 }, { 0 }, { 0 }, { 0 } }, /* stat counter reserved */ \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, /* malloc_bins */ \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, /* page bins */ \ + { MI_INIT5(MI_STAT_COUNT_NULL) } /* chunk bins */ + +#define MI_STATS_NULL \ + { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_FIELDS_NULL } // -------------------------------------------------------- // Statically allocate an empty theap as the initial @@ -89,21 +105,11 @@ const mi_page_t _mi_page_empty = { // may lead to allocation itself on some platforms) // -------------------------------------------------------- -static mi_decl_cache_align mi_subproc_t subproc_main -#if __cplusplus - = { }; // empty initializer to prevent running the constructor (with msvc) -#else - = { 0 }; // C zero initialize -#endif - -static mi_subproc_t* subprocs = &subproc_main; -static mi_lock_t subprocs_lock; - -static mi_decl_cache_align mi_tld_t tld_empty = { - 0, // thread_id +static mi_decl_cache_align mi_tld_t mi_tld_detached = { + MI_THREADID_DETACHED, // thread_id 0, // thread_seq 0, // default numa node - &subproc_main, // subproc + NULL, // subproc NULL, // theaps list MI_LOCK_INITIALIZER, // theaps lock false, // recurse @@ -111,13 +117,13 @@ static mi_decl_cache_align mi_tld_t tld_empty = { MI_MEMID_STATIC // memid }; -mi_decl_cache_align const mi_theap_t _mi_theap_empty = { - &tld_empty, // tld +mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { + MI_SMALL_PAGES_EMPTY, // direct small pages + &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap + MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount - MI_ATOMIC_VAR_INIT(0), // freed 0, // heartbeat - 0, // cookie { {0}, {0}, 0, true }, // random 0, // page count MI_BIN_FULL, 0, // page retired min/max @@ -128,154 +134,42 @@ mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, // full page retain false, // allow reclaim true, // allow abandon + true, // is_detached #if MI_GUARDED - 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`) + 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif - MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, - { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats + MI_STATS_NULL, // stats }; -mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = { - &tld_empty, // tld - MI_ATOMIC_VAR_INIT(NULL), // heap - MI_ATOMIC_VAR_INIT(1), // refcount - MI_ATOMIC_VAR_INIT(0), // freed - 0, // heartbeat - 0, // cookie - { {0}, {0}, 0, true }, // random - 0, // page count - MI_BIN_FULL, 0, // page retired min/max - 0, // pages_full_size - 0, 0, // generic count - NULL, NULL, // tnext, tprev - NULL, NULL, // hnext, hprev - 0, // full page retain - false, // allow reclaim - true, // allow abandon - #if MI_GUARDED - 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`) - #endif - MI_SMALL_PAGES_EMPTY, - MI_PAGE_QUEUES_EMPTY, - MI_MEMID_STATIC, - { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats -}; - -// Heap for the main thread - -#define MI_THREADID_INVALID ((mi_threadid_t)(~0)) - -extern mi_decl_hidden mi_decl_cache_align mi_theap_t theap_main; -extern mi_decl_hidden mi_decl_cache_align mi_heap_t heap_main; - -static mi_decl_cache_align mi_tld_t tld_main = { - 0, // thread_id - 0, // thread_seq - 0, // numa node - &subproc_main, // subproc - &theap_main, // theaps list - MI_LOCK_INITIALIZER, // theaps lock - false, // recurse - false, // is_in_threadpool - MI_MEMID_STATIC // memid -}; - -mi_decl_cache_align mi_theap_t theap_main = { - &tld_main, // thread local data - MI_ATOMIC_VAR_INIT(&heap_main), // main heap - MI_ATOMIC_VAR_INIT(1), // refcount - MI_ATOMIC_VAR_INIT(0), // freed - 0, // heartbeat - 0, // initial cookie - { {0x846ca68b}, {0}, 0, true }, // random - 0, // page count - MI_BIN_FULL, 0, // page retired min/max - 0, // pages_full_size - 0, 0, // generic count - NULL, NULL, // tnext, tprev - NULL, NULL, // hnext, hprev - 2, // full page retain - true, // allow page reclaim - true, // allow page abandon - #if MI_GUARDED - 0, 0, 0, 0, - #endif - MI_SMALL_PAGES_EMPTY, - MI_PAGE_QUEUES_EMPTY, - MI_MEMID_STATIC, - { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats -}; - -mi_decl_cache_align mi_heap_t heap_main -#if __cplusplus - = { }; // empty initializer to prevent running the constructor (with msvc) -#else - = { 0 }; // C zero initialize -#endif +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER -// the theap belonging to the main heap -mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main = NULL; -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { - const mi_threadid_t tid = _mi_prim_thread_id(); - mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits - return tid; -} +// pre-allocate the process heap, and meta-data theap +static mi_decl_cache_align mi_heap_t mi_process_heap_main = mi_init_struct_zero; +static mi_decl_cache_align mi_theap_t mi_process_theap_meta = mi_init_struct_zero; -#if MI_TLS_MODEL_THREAD_LOCAL -// the thread-local main theap for allocation -mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty; -// the last used non-main theap -mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty; -#endif +// pre-allocate the initial tld and theap for the main thread (this is not strictly needed but nice for stats) +static mi_decl_cache_align mi_tld_t mi_process_tld_main = mi_init_struct_zero; +static mi_decl_cache_align mi_theap_t mi_process_theap_main = mi_init_struct_zero; -bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. +mi_decl_hidden mi_decl_cache_align mi_theap_t _mi_theap_empty_wrong = mi_init_struct_zero; // used for error paths +mi_decl_hidden bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. -mi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }; -#undef MI_STAT_COUNT -#undef MI_STAT_COUNTER - - -#if MI_GUARDED -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = sample_rate; - theap->guarded_sample_count = sample_rate; // count down samples - if (theap->guarded_sample_rate > 1) { - if (seed == 0) { - seed = _mi_theap_random_next(theap); - } - theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` - } +mi_page_t* _mi_page_empty_get(void) { + return (mi_page_t*)&mi_page_empty; } -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - theap->guarded_size_min = min; - theap->guarded_size_max = (min > max ? min : max); -} - -void _mi_theap_guarded_init(mi_theap_t* theap) { - mi_theap_guarded_set_sample_rate(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), - (size_t)mi_option_get(mi_option_guarded_sample_seed)); - mi_theap_guarded_set_size_bound(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), - (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); -} -#else -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) { + return (mi_theap_t*)&_mi_theap_empty; } -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); -} -void _mi_theap_guarded_init(mi_theap_t* theap) { - MI_UNUSED(theap); +bool _mi_is_empty_theap(const mi_theap_t* theap) { + return (theap == &_mi_theap_empty); } -#endif /* ----------------------------------------------------------- Initialization @@ -283,361 +177,210 @@ void _mi_theap_guarded_init(mi_theap_t* theap) { can cause allocation and induce recursion during initialization. ----------------------------------------------------------- */ +static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc); -// Initialize main subproc -static void mi_subproc_main_init(void) { - if (subproc_main.memid.memkind != MI_MEM_STATIC) { - subproc_main.memid = _mi_memid_create(MI_MEM_STATIC); - subproc_main.heaps = &heap_main; - subproc_main.heap_total_count = 1; - subproc_main.heap_count = 1; - mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &heap_main); - __mi_stat_increase_mt(&subproc_main.stats.heaps, 1); - mi_stats_header_init(&subproc_main.stats); - mi_lock_init(&subproc_main.arena_reserve_lock); - mi_lock_init(&subproc_main.heaps_lock); - mi_lock_init(&subprocs_lock); - mi_lock_init(&tld_empty.theaps_lock); - } -} +// Initialize main heap +static void mi_heap_main_init_once(void) { + mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC); + _mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong)); -// Initialize main tld -static void mi_tld_main_init(void) { - if (tld_main.thread_id == 0) { - tld_main.thread_id = _mi_prim_thread_id(); - mi_lock_init(&tld_main.theaps_lock); - } -} + // initialize the main subprocess + mi_subproc_t* subproc_main = _mi_subproc_main_init(); -void _mi_theap_options_init(mi_theap_t* theap) { - theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); - theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); - theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); -} + // detached tld for mi_theap_empty (and theap_meta) + mi_tld_detached.memid = memid_static; + mi_tld_init(&mi_tld_detached, 0, subproc_main); -// Initialization of the (statically allocated) main theap, and the main tld and subproc. -static void mi_theap_main_init(void) { - if mi_unlikely(theap_main.memid.memkind != MI_MEM_STATIC) { - // theap - theap_main.memid = _mi_memid_create(MI_MEM_STATIC); - #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB) - _mi_random_init_weak(&theap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) - #else - _mi_random_init(&theap_main.random); - #endif - theap_main.cookie = _mi_theap_random_next(&theap_main); - _mi_theap_options_init(&theap_main); - _mi_theap_guarded_init(&theap_main); - } -} + // main process heap + mi_process_heap_main.memid = memid_static; + mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main); + _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0); -// Initialize main heap -static void mi_heap_main_init(void) { - if mi_unlikely(heap_main.subproc == NULL) { - heap_main.subproc = &subproc_main; - heap_main.theaps = &theap_main; + // detached theap for allocating meta-data (we can allocate on this without having an initialized thread) + mi_process_theap_meta.memid = memid_static; + _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); + mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads + mi_process_theap_meta.page_full_retain = 2; + subproc_main->theap_meta = &mi_process_theap_meta; - mi_theap_main_init(); - mi_subproc_main_init(); - mi_tld_main_init(); + // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` +} - mi_lock_init(&heap_main.theaps_lock); - mi_lock_init(&heap_main.os_abandoned_pages_lock); - mi_lock_init(&heap_main.arena_pages_lock); +static void mi_heap_main_init(void) { + mi_atomic_do_once { + mi_heap_main_init_once(); } } +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { + mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); + if mi_likely(heap!=NULL) { + return heap; + } + else if (_mi_subproc_is_main(subproc)) { + mi_heap_main_init(); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL); + return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); + } + else { + mi_assert_internal(false); + return &mi_process_heap_main; + } +} /* ----------------------------------------------------------- Thread local data ----------------------------------------------------------- */ -// Allocate fresh tld -static mi_tld_t* mi_tld_alloc(void) { - if (_mi_is_main_thread()) { - mi_atomic_increment_relaxed(&tld_main.subproc->thread_count); - return &tld_main; +static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) { + tld->subproc = subproc; + tld->theaps = NULL; + mi_lock_init(&tld->theaps_lock); + if (tld->thread_id == MI_THREADID_DETACHED) { + tld->numa_node = -1; } else { - // allocate tld meta-data - // note: we need to be careful to not access the tld from `_mi_meta_zalloc` - // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`). - mi_memid_t memid; - mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(sizeof(mi_tld_t), &memid); - if (tld==NULL) { - _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n"); - return NULL; - } - tld->memid = memid; - tld->theaps = NULL; - mi_lock_init(&tld->theaps_lock); - tld->subproc = &subproc_main; tld->numa_node = _mi_os_numa_node(); tld->thread_id = _mi_prim_thread_id(); - tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count); tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool(); + tld->thread_seq = tseq; mi_atomic_increment_relaxed(&tld->subproc->thread_count); - return tld; - } -} - -#define MI_TLD_INVALID ((mi_tld_t*)1) - -mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { - if (tld==NULL || tld==MI_TLD_INVALID) return; - mi_atomic_decrement_relaxed(&tld->subproc->thread_count); - tld->thread_id = MI_THREADID_INVALID; // note: not 0 as that would re-initialize tld_main - // we also need to set an invalid tid for tld_main as sometimes the same thread-id - // is reused by the OS after a thread has terminated. (see issue #1287) - mi_lock_done(&tld->theaps_lock); - _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid); // note: safe for static tld_main -} - -// return the thread local heap ensuring it is initialized (and not `NULL` or `&_mi_theap_empty`); -mi_theap_t* _mi_theap_default_safe(void) { - mi_theap_t* theap = _mi_theap_default(); - if mi_likely(mi_theap_is_initialized(theap)) return theap; - mi_thread_init(); - mi_assert_internal(mi_theap_is_initialized(_mi_theap_default())); - return _mi_theap_default(); -} - -// return the main theap ensuring it is initialized. -mi_theap_t* _mi_theap_main_safe(void) { - mi_theap_t* theap = __mi_theap_main; - if mi_unlikely(theap==NULL) { // if thread_init or default_set was never called - mi_thread_init(); // sets the default slot to the main theap - theap = _mi_theap_default(); - mi_assert_internal(theap!=NULL); - mi_assert_internal(_mi_is_theap_main(theap)); - if (_mi_is_theap_main(theap)) { - __mi_theap_main = theap; - } } - mi_assert_internal(theap!=NULL && _mi_is_theap_main(theap)); - return theap; + return tld; } +// Allocate fresh tld +static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) { + mi_assert_internal(subproc->theap_meta != NULL); // should be initialized on the main thread before other threads allocate + const size_t tseq = mi_atomic_increment_relaxed(&subproc->thread_total_count); -mi_subproc_t* _mi_subproc_main(void) { - return &subproc_main; -} - -mi_subproc_t* _mi_subproc(void) { - // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()` - // todo: this will still fail on OS systems where the first access to a thread-local causes allocation. - // on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being - // stored in a TLS slot for example) - mi_theap_t* theap = _mi_theap_default(); - if (theap == NULL || theap->tld == NULL) { // see issue #1289 - return _mi_subproc_main(); + mi_memid_t memid; + mi_tld_t* tld; + if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) { + tld = &mi_process_tld_main; + memid = _mi_memid_create_static(tld,sizeof(*tld)); } else { - return theap->tld->subproc; // avoid using thread local storage (`thread_tld`) + tld = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid); } -} - -mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { - mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); - if mi_likely(heap!=NULL) { - return heap; - } - else { - mi_heap_main_init(); - mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL); - return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); + if (tld==NULL) { + _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n"); + return NULL; } + tld->memid = memid; + return mi_tld_init(tld,tseq,subproc); } -mi_heap_t* mi_heap_main(void) { - return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init -} - -bool _mi_is_heap_main(const mi_heap_t* heap) { - mi_assert_internal(heap!=NULL); - return (_mi_subproc_heap_main(heap->subproc) == heap); +mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { + if (tld==NULL) return; + mi_atomic_decrement_relaxed(&tld->subproc->thread_count); + tld->thread_id = (mi_threadid_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id + // is reused by the OS after a thread has terminated. (see issue #1287) + mi_lock_done(&tld->theaps_lock); + _mi_meta_free(tld->subproc, tld, tld->memid); // note: safe for static tld } -bool _mi_is_theap_main(const mi_theap_t* theap) { - return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap))); -} /* ----------------------------------------------------------- - Sub process + Thread Init ----------------------------------------------------------- */ - -mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { - return (mi_subproc_t*)(subproc_id._mi_subproc_id); -} - -mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) { - mi_subproc_id_t id = { subproc }; - return id; -} - -mi_subproc_id_t mi_subproc_main(void) { - return _mi_subproc_to_id(_mi_subproc_main()); -} - -mi_subproc_id_t mi_subproc_current(void) { - return _mi_subproc_to_id(_mi_subproc()); -} - -mi_subproc_id_t mi_subproc_new(void) { - static _Atomic(size_t) subproc_total_count; - mi_memid_t memid; - mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid); - if (subproc == NULL) return _mi_subproc_to_id(NULL); - subproc->memid = memid; - subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1; - mi_stats_header_init(&subproc->stats); - mi_lock_init(&subproc->arena_reserve_lock); - mi_lock_init(&subproc->heaps_lock); - mi_lock(&subprocs_lock) { - // push on subproc list - subproc->next = subprocs; - if (subprocs!=NULL) { subprocs->prev = subproc; } - subprocs = subproc; +#if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) +static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) { + const mi_threadid_t tid = _mi_thread_id(); + mi_theap_t* thread_theap = NULL; + mi_lock(&heap->theaps_lock) { + for(mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext ) { + if (theap->tld->thread_id == tid) { + thread_theap = theap; + break; + } + } } - return _mi_subproc_to_id(subproc); + return thread_theap; } +#endif -// destroy all subproc resources including arena's, heap's etc. -static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock) +// Initialize thread +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) { - if (subproc==NULL) return; + // ensure our process has started already + mi_process_init(); - // remove from the subproc list - mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) { - if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; } - if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; } - else { mi_assert_internal(subprocs==subproc); subprocs = subproc->next; } - } + // if the theap_default is already set we have already initialized + mi_theap_t* theap = _mi_theap_default(); + if (mi_theap_is_initialized(theap)) return theap; + + // initialize the default theap + // note: we cannot access thread-locals yet as that can cause (recursive) allocation + // (on macOS <= 14 for example where the loader allocates thread-local data on demand). + if (heap_main==NULL) { + heap_main = mi_heap_main(); + mi_assert_internal(heap_main == &mi_process_heap_main); + } + mi_assert_internal(heap_main!=NULL); + + #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) + theap = mi_heap_check_for_existing_theap(heap_main); // recursion check + #if !defined(MI_TLS_RECURSE_GUARD) + mi_assert_internal(theap==NULL); + #endif + #else + theap = NULL; + #endif - // destroy all subproc heaps - mi_lock(&subproc->heaps_lock) { - mi_heap_t* heap = subproc->heaps; - while (heap != NULL) { - mi_heap_t* next = heap->next; - if (heap!=subproc->heap_main) { mi_heap_destroy(heap); } - heap = next; + if (theap==NULL) { + // allocated the tld + mi_tld_t* tld = mi_tld_create(heap_main->subproc); + if (tld==NULL) return NULL; // out-of-memory on tld allocation + // allocate and initialize the theap for the main heap + if (tld==&mi_process_tld_main) { + theap = &mi_process_theap_main; // initial theap is pre-allocated + theap->memid = _mi_memid_create_static(theap,sizeof(*theap)); + } + else { + theap = _mi_theap_alloc(heap_main,tld); // otherwise meta allocate + if (theap==NULL) { mi_tld_free(tld); return NULL; } // out-of-memory on theap allocation } - mi_assert_internal(subproc->heaps == subproc->heap_main); - _mi_heap_force_destroy(subproc->heap_main); // no warning if destroying the main heap + _mi_theap_init(theap,heap_main,tld); } - // remove associated arenas - _mi_arenas_unsafe_destroy_all(subproc); + // now initialize the thread + _mi_theap_default_set(theap); + // and only then set the heap_theap field as that accesses thread locals + _mi_heap_theap_set(heap_main, theap); // todo: can fail! - // merge stats back into the main subproc? - if (subproc!=&subproc_main) { - _mi_stats_merge_into(&subproc_main.stats, &subproc->stats); - } + mi_assert_internal(mi_theap_is_initialized(theap)); + mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap)); + mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap); - // safe to release - // todo: should we refcount subprocesses? - mi_lock_done(&subproc->arena_reserve_lock); - mi_lock_done(&subproc->heaps_lock); - if (subproc!=&subproc_main) { - _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid); - } - else { - // for the main subproc, also release the global page map - _mi_page_map_unsafe_destroy(&subproc_main); - } -} - -void mi_subproc_destroy(mi_subproc_id_t subproc_id) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL || subproc==&subproc_main) return; - mi_subproc_unsafe_destroy(subproc, true /* take lock */); + mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? + // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); + return theap; } -static void mi_subprocs_unsafe_destroy_all(void) { - mi_lock(&subprocs_lock) { - mi_subproc_t* subproc = subprocs; - while (subproc!=NULL) { - mi_subproc_t* next = subproc->next; - if (subproc!=&subproc_main) { - mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */); - } - subproc = next; - } - } - mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */); +mi_theap_t* _mi_thread_init(void) { + return _mi_thread_init_with_heap(NULL); } - -void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - mi_tld_t* const tld = _mi_theap_default_safe()->tld; - mi_assert(tld->subproc== &subproc_main); - if (tld->subproc != &subproc_main) { - _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (id: %p)\n", subproc); - return; - } - tld->subproc = subproc; - tld->thread_seq = mi_atomic_increment_relaxed(&subproc->thread_total_count); - mi_atomic_decrement_relaxed(&subproc_main.thread_count); - mi_atomic_increment_relaxed(&subproc->thread_count); +void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept { + _mi_thread_init(); } -bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL) return false; - bool ok = true; - mi_lock(&subproc->heaps_lock) { - for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) { - ok = (*visitor)(heap, arg); - } - } - return ok; -} - /* ----------------------------------------------------------- - Allocate theap data + Theaps done ----------------------------------------------------------- */ -// Initialize the thread local default theap, called from `mi_thread_init` -static mi_theap_t* _mi_thread_init_theap_default(void) { - mi_theap_t* theap = _mi_theap_default(); - if (mi_theap_is_initialized(theap)) return theap; - if (_mi_is_main_thread()) { - mi_heap_main_init(); - theap = &theap_main; - } - else { - // allocates tld data - // note: we cannot access thread-locals yet as that can cause (recursive) allocation - // (on macOS <= 14 for example where the loader allocates thread-local data on demand). - mi_tld_t* tld = mi_tld_alloc(); - if (tld==NULL) return NULL; // things are very wrong if this fails (out of memory) - // allocate and initialize the theap for the main heap - theap = _mi_theap_create(mi_heap_main(), tld); - } - // associate the theap with this thread - // (this is safe, on macOS for example, the theap is set in a dedicated TLS slot and thus does not cause recursive allocation) - _mi_theap_default_set(theap); - return theap; -} - - // Free the thread local theaps static void mi_thread_theaps_done(mi_tld_t* tld) { - // reset the thread local theaps - _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty); - _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty); - __mi_theap_main = NULL; - // abandon the pages of all theaps in this thread mi_lock(&tld->theaps_lock) { mi_theap_t* theap = tld->theaps; while (theap != NULL) { - mi_theap_t* next = theap->tnext; + mi_theap_t* next = theap->tnext; // never destroy theaps; if a dll is linked statically with mimalloc, // there may still be delete/free calls after the mi_fls_done is called. Issue #207 _mi_theap_collect_abandon(theap); @@ -646,39 +389,38 @@ static void mi_thread_theaps_done(mi_tld_t* tld) } } - // free the theaps of this thread. - // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically. - // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock - // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list. - bool all_freed; - do { - all_freed = true; - mi_lock(&tld->theaps_lock) { - mi_theap_t* theap = tld->theaps; - while (theap != NULL) { - mi_theap_t* next = theap->tnext; - mi_assert_internal(theap->page_count==0); - if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) { - all_freed = false; - } - theap = next; - } - } - if (!all_freed) { - mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); - _mi_prim_thread_yield(); - } - else { - mi_assert_internal(tld->theaps==NULL); + // reset the thread local theaps + // note: do this after abandon as page->heap may be NULL and mi_heap_main should return the heap + // belonging to the right subprocess + _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty); + _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty); + + // We might run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically. + // we first detach our theaps list from any heaps + _mi_tld_detach_theaps(tld); + + // no heaps point to our theaps anymore, free them + mi_lock(&tld->theaps_lock) { // paranoia + mi_theap_t* theap = tld->theaps; + tld->theaps = NULL; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + mi_assert_internal(theap->page_count==0); + mi_assert_internal(_mi_theap_heap_peek(theap)==NULL); + theap->tld = NULL; + theap->tnext = NULL; + theap->tprev = NULL; + mi_assert_internal(mi_atomic_load_relaxed(&theap->refcount) == 1); // as the cached entry is set to empty + _mi_theap_decref(theap); + theap = next; } - } while (!all_freed); + } mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete mi_assert(!mi_theap_is_initialized(_mi_theap_default())); } - // -------------------------------------------------------- // Try to run `mi_thread_done()` automatically so any memory // owned by the thread but not yet released can be abandoned @@ -695,35 +437,13 @@ static void mi_thread_theaps_done(mi_tld_t* tld) // to set up the thread local keys. // -------------------------------------------------------- -// Set up handlers so `mi_thread_done` is called automatically +// Set up hooks so `mi_thread_done` is called automatically static void mi_process_setup_auto_thread_done(void) { mi_atomic_do_once { _mi_prim_thread_init_auto_done(); - _mi_theap_default_set(&theap_main); } } - -bool _mi_is_main_thread(void) { - return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id()); -} - - -// Initialize thread -void mi_thread_init(void) mi_attr_noexcept -{ - // ensure our process has started already - mi_process_init(); - // if the theap_default is already set we have already initialized - if (_mi_thread_is_initialized()) return; - - // initialize the default theap - if (_mi_thread_init_theap_default() == NULL) return; // out-of-memory on tld/theap allocation - - mi_heap_stat_increase(mi_heap_main(), threads, 1); - // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); -} - void mi_thread_done(void) mi_attr_noexcept { _mi_thread_done(NULL); } @@ -732,11 +452,7 @@ void _mi_thread_done(mi_theap_t* _theap_main) { // NULL can be passed on some platforms if (_theap_main==NULL) { - _theap_main = __mi_theap_main; // don't call `mi_theap_main_safe` as that re-initializes the thread - if (_theap_main==NULL) { // can happen if `mi_theap_main_safe` is never called; but then the default is main - _theap_main = _mi_theap_default(); - mi_assert_internal(_theap_main==NULL || _mi_is_theap_main(_theap_main)); - } + _theap_main = _mi_theap_default(); } // prevent re-entrancy through theap_done/theap_set_default_direct (issue #699) @@ -744,14 +460,14 @@ void _mi_thread_done(mi_theap_t* _theap_main) return; } + // get the current tld + mi_tld_t* const tld = _theap_main->tld; + // release dynamic thread_local's _mi_thread_locals_thread_done(); - // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage) - mi_tld_t* const tld = _theap_main->tld; - // adjust stats - mi_heap_stat_decrease(_mi_subproc_heap_main(tld->subproc), threads, 1); // todo: or `_theap_main->heap`? + mi_subproc_stat_decrease(tld->subproc, threads, 1); // todo: or `_theap_main->heap`? // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps... if (tld->thread_id != _mi_prim_thread_id()) return; @@ -763,226 +479,16 @@ void _mi_thread_done(mi_theap_t* _theap_main) mi_tld_free(tld); } - -mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) { - return (mi_theap_t*)&_mi_theap_empty; -} - -#if MI_TLS_MODEL_DYNAMIC_WIN32 - -// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) -// See for the offsets. -#if MI_SIZE_SIZE==4 -#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE) -#else -#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE) -#endif -#define MI_TLS_DIRECT_SLOTS (64) -#define MI_TLS_EXPANSION_SLOTS (1024) - -#if !MI_WIN_DIRECT_TLS -// we initially use the last of the expansion slots as the default NULL. -// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc :-( (but this is quite unlikely) -#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT -#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) -#else -// with only direct entries, use the "arbitrary user data" field -// and assume it is NULL (see also ) -#define MI_TLS_INITIAL_SLOT (5) -#define MI_TLS_INITIAL_EXPANSION_SLOT (0) -#endif - -mi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT; -mi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT; -mi_decl_hidden size_t _mi_theap_cached_slot = MI_TLS_INITIAL_SLOT; -mi_decl_hidden size_t _mi_theap_cached_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT; - -static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES; -static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES; - -static bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) { - const DWORD index = TlsAlloc(); - *raw_index = index; - if (index==TLS_OUT_OF_INDEXES) { - *extended = 0; - *slot = 0; - return false; - } - else if (index= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT); - if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) { - mi_prim_tls_slot_set(slot, value); - } - else { - mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS); - TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed - } -} - -#elif MI_TLS_MODEL_DYNAMIC_PTHREADS - -// only for pthreads for now -mi_decl_hidden pthread_key_t _mi_theap_default_key = 0; -mi_decl_hidden pthread_key_t _mi_theap_cached_key = 0; - -// create a non-zero pthread key -static int mi_pthread_key_create( pthread_key_t* pkey ) { - pthread_key_t key; - int err = pthread_key_create(&key, NULL); - if (err!=0) return err; - if (key==0) { - // if we get a zero key, create another one as we use 0 for an invalid key - pthread_key_t key2; - err = pthread_key_create(&key2, NULL); - pthread_key_delete(key); // delete the old key - if (err!=0) return err; - key = key2; - } - mi_assert_internal(key!=0); - *pkey = key; - return 0; -} - -static void mi_tls_slots_init(void) { - mi_atomic_do_once { - int err = mi_pthread_key_create(&_mi_theap_default_key); - if (err==0) { - err = mi_pthread_key_create(&_mi_theap_cached_key); - } - if (err!=0) { - _mi_error_message(EFAULT, "unable to allocate pthread keys (error %d)\n", err); - } - } -} - -static void mi_tls_slots_done(void) { - if (_mi_theap_default_key != 0) { - pthread_key_delete(_mi_theap_default_key); - _mi_theap_default_key = 0; - } - if (_mi_theap_cached_key != 0) { - pthread_key_delete(_mi_theap_cached_key); - _mi_theap_cached_key = 0; - } -} - -#else - -static void mi_tls_slots_init(void) { - // nothing -} - -static void mi_tls_slots_done(void) { - // nothing -} - -#endif - -void _mi_theap_cached_set(mi_theap_t* theap) { - mi_theap_t* prev = _mi_theap_cached(); - if (prev==theap) return; - // set - mi_tls_slots_init(); - #if MI_TLS_MODEL_THREAD_LOCAL - __mi_theap_cached = theap; - #elif MI_TLS_MODEL_FIXED_SLOT - mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_CACHED, theap); - #elif MI_TLS_MODEL_DYNAMIC_WIN32 - mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap); - #elif MI_TLS_MODEL_DYNAMIC_PTHREADS - if (_mi_theap_cached_key!=0) pthread_setspecific(_mi_theap_cached_key, theap); - #endif - // update refcounts (so cached theap memory keeps available until no longer cached) - _mi_theap_incref(theap); - _mi_theap_decref(prev); -} - -void _mi_theap_default_set(mi_theap_t* theap) { - mi_theap_t* const theap_old = _mi_theap_default(); - mi_assert_internal(theap != NULL); - mi_assert_internal(theap->tld != NULL); - mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id()); - mi_tls_slots_init(); - #if MI_TLS_MODEL_THREAD_LOCAL - __mi_theap_default = theap; - #elif MI_TLS_MODEL_FIXED_SLOT - mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_DEFAULT, theap); - #elif MI_TLS_MODEL_DYNAMIC_WIN32 - mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap); - #elif MI_TLS_MODEL_DYNAMIC_PTHREADS - if (_mi_theap_default_key!=0) pthread_setspecific(_mi_theap_default_key, theap); - #endif - - // set theap main if needed - if (mi_theap_is_initialized(theap)) { - // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again) - _mi_prim_thread_associate_default_theap(theap); - if (_mi_is_heap_main(_mi_theap_heap(theap))) { - __mi_theap_main = theap; - } - } - - // ensure either the default slot contains the main theap, or __mi_theap_main is initialized - if (mi_theap_is_initialized(theap_old) && _mi_is_heap_main(_mi_theap_heap(theap_old))) { - __mi_theap_main = theap_old; - } -} - void mi_thread_set_in_threadpool(void) mi_attr_noexcept { - mi_theap_t* theap = _mi_theap_default_safe(); + mi_theap_t* theap = mi_theap_get_default(); theap->tld->is_in_threadpool = true; } + // -------------------------------------------------------- -// Run functions on process init/done, and thread init/done +// Process init and done // -------------------------------------------------------- + static bool os_preloading = true; // true until this module is initialized // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. @@ -995,20 +501,13 @@ mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { return _mi_is_redirected(); } -// Called once by the process loader from `src/prim/prim.c` +// Called once by the process loader from `src/prim/prim.c` before `main` is called. void _mi_auto_process_init(void) { - // mi_heap_main_init(); - // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) - // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true; - // if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697) - // #endif - os_preloading = false; - mi_assert_internal(_mi_is_main_thread()); mi_process_init(); mi_process_setup_auto_thread_done(); - _mi_thread_locals_init(); + _mi_options_post_init(); // now we can print to stderr if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); @@ -1020,97 +519,41 @@ void _mi_auto_process_init(void) { } // reseed random - _mi_random_reinit_if_weak(&theap_main.random); -} - -// CPU features -mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast -mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast -mi_decl_cache_align bool _mi_cpu_has_popcnt = false; - -#if (MI_ARCH_X64 || MI_ARCH_X86) -#if defined(__GNUC__) -// #include -static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { - // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx) - // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect). - uint32_t eax, ebx, ecx, edx; - __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : ); - regs4[0] = eax; - regs4[1] = ebx; - regs4[2] = ecx; - regs4[3] = edx; - return true; -} - -#elif defined(_MSC_VER) -static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { - __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel); - return true; -} -#else -static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { - MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel); - return false; -} -#endif - -static void mi_detect_cpu_features(void) { - // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) - // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) - // FSRS for fast short rep stosb - bool amd = false; - bool fsrm = false; - // bool erms = false; - bool fsrs = false; - uint32_t cpu_info[4]; - if (mi_cpuid(cpu_info, 0, 0)) { - amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) - } - if (mi_cpuid(cpu_info, 7, 0)) { - fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see - // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see - } - if (mi_cpuid(cpu_info, 7, 1)) { - fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see - } - if (mi_cpuid(cpu_info, 1, 0)) { - _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see - } - - if (fsrm) { - _mi_cpu_movsb_max = 127; - } - if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 - _mi_cpu_stosb_max = 127; + mi_theap_t* theap = _mi_theap_default(); + if (theap != NULL) { + _mi_random_reinit_if_weak(&theap->random); + mi_subproc_t* subproc = _mi_theap_subproc(theap); + if (subproc->theap_meta != NULL) { + mi_lock(&subproc->theap_meta_lock) { + _mi_random_reinit_if_weak(&subproc->theap_meta->random); + } + } } } -#else -static void mi_detect_cpu_features(void) { - #if MI_ARCH_ARM64 - _mi_cpu_has_popcnt = true; + +// Initialize the process; called by thread_init, the process loader, or an initial allocation (perhaps by the loader or a system library) +static void mi_process_init_once(void) { + #if defined(__CYGWIN__) // we need to kickstart the cygwin runtime + __mi_thread_id_helper = NULL; #endif -} -#endif + _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + _mi_detect_cpu_features(); + _mi_options_init(); // read environment (if possible) + _mi_stats_init(); // start timer + _mi_os_init(); // primitive dependent -// Initialize the process; called by thread_init or the process loader -static void mi_process_init_once(void) mi_attr_noexcept { - _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + mi_heap_main_init(); // before page_map_init so stats are working + _mi_page_map_init(); // todo: this could fail.. should we abort in that case? + mi_thread_init(); - mi_detect_cpu_features(); - _mi_options_init(); - _mi_stats_init(); - _mi_os_init(); // the following can potentially allocate (on freeBSD for pthread keys) - // todo: do 2-phase so we can use stats at first, then later init the keys? - mi_heap_main_init(); // before page_map_init so stats are working - _mi_page_map_init(); // todo: this could fail.. should we abort in that case? - mi_thread_init(); + _mi_tls_slots_init(); // pthread key create + _mi_thread_locals_init(); // pthread key create _mi_process_is_initialized = true; - - #if defined(_WIN32) && defined(MI_WIN_USE_FLS) + + #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup // will not call _mi_thread_done on the (still executing) main thread. See issue #508. @@ -1138,16 +581,13 @@ static void mi_process_init_once(void) mi_attr_noexcept { // Initialize the process; called by thread_init or the process loader void mi_process_init(void) mi_attr_noexcept { - // #if _MSC_VER < 1920 - // mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main - // #endif mi_atomic_do_once { mi_process_init_once(); } } -// Called when the process is done +// Called when the process is done static void mi_process_done_once(void) { // only shutdown if we were initialized if (!_mi_process_is_initialized) return; @@ -1156,8 +596,8 @@ static void mi_process_done_once(void) { if (process_done) return; process_done = true; - // free dynamic thread locals (if used at all) - _mi_thread_locals_done(); + // decref any cached theap + _mi_theap_cached_set(_mi_theap_empty_get()); // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread _mi_prim_thread_done_auto_done(); @@ -1177,33 +617,41 @@ static void mi_process_done_once(void) { // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free // since after process_done there might still be other code running that calls `free` (like at_exit routines, // or C-runtime termination code. + mi_subproc_t* subproc_main = _mi_subproc_main(); if (mi_option_is_enabled(mi_option_destroy_on_exit)) { - mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map! + _mi_subprocs_unsafe_destroy_all(); // destroys all mi_subprocs, arenas, thread locals, and the page_map! } else { - mi_heap_stats_merge_to_subproc(mi_heap_main()); - } - - // careful now to no longer access any allocator functionality - if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { - mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + // free dynamic thread locals (if used at all) + _mi_thread_locals_thread_done(); + _mi_thread_locals_done(); + if (subproc_main->heap_main != NULL) { + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + _mi_theap_merge_stats(subproc_main->theap_meta); + _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free + mi_heap_stats_merge_to_subproc(subproc_main->heap_main); + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek + } + } } - mi_lock_done(&subprocs_lock); - mi_tls_slots_done(); + + _mi_tls_slots_done(); + _mi_subproc_main_done(); _mi_allocator_done(); - _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id); + _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id); os_preloading = true; // don't call the C runtime anymore } -// Called when the process is done (cdecl as it is used with `at_exit` on some platforms) +// Call when the process is done (cdecl as it is used with `at_exit` on some platforms) void mi_cdecl mi_process_done(void) mi_attr_noexcept { mi_atomic_do_once { mi_process_done_once(); } } +// Called automatically when the process is done (cdecl as it is used with `at_exit` on some platforms) void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept { - if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return; + if (_mi_option_get_fast(mi_option_destroy_on_exit)>=2) return; // allow disabling auto process done mi_process_done(); } diff --git a/system/lib/mimalloc/src/libc.c b/system/lib/mimalloc/src/libc.c index 5a5e586e9613a..17dd1d1dcab5d 100644 --- a/system/lib/mimalloc/src/libc.c +++ b/system/lib/mimalloc/src/libc.c @@ -1,12 +1,12 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2024, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ // -------------------------------------------------------- -// This module defines various std libc functions to reduce +// This module defines various standard libc functions to reduce // the dependency on libc, and also prevent errors caused // by some libc implementations when called before `main` // executes (due to malloc redirection) @@ -39,8 +39,8 @@ bool _mi_streq(const char* s, const char* t) { return (*s == *t); } -void _mi_strlcpy(char* dest, const char* src, size_t dest_size) { - if (dest==NULL || src==NULL || dest_size == 0) return; +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0); // copy until end of src, or when dest is (almost) full while (*src != 0 && dest_size > 1) { *dest++ = *src++; @@ -48,17 +48,18 @@ void _mi_strlcpy(char* dest, const char* src, size_t dest_size) { } // always zero terminate *dest = 0; + return (*src == 0); } -void _mi_strlcat(char* dest, const char* src, size_t dest_size) { - if (dest==NULL || src==NULL || dest_size == 0) return; +bool _mi_strlcat(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0); // find end of string in the dest buffer while (*dest != 0 && dest_size > 1) { dest++; dest_size--; } // and catenate - _mi_strlcpy(dest, src, dest_size); + return _mi_strlcpy(dest, src, dest_size); } size_t _mi_strnlen(const char* s, size_t max_len) { @@ -76,7 +77,7 @@ char* _mi_strnstr(char* s, size_t max_len, const char* pat) { if (s==NULL) return NULL; if (pat==NULL) return s; const size_t m = _mi_strnlen(s, max_len); - const size_t n = _mi_strlen(pat); + const size_t n = _mi_strlen(pat); for (size_t start = 0; start + n <= m; start++) { size_t i = 0; while (i 0 ? 0 : (res == 0 ? ENOENT : EAGAIN)); @@ -118,7 +119,7 @@ bool _mi_atomic_once_enter(mi_atomic_once_t* once) { } const mi_threadid_t current_tid = _mi_thread_id(); if (once_tid == current_tid) { - return false; // recursive invocation; we need this for process_init for example + return false; // recursive invocation; don't block on ourselves } mi_lock_acquire(&once->lock); @@ -139,6 +140,96 @@ void _mi_atomic_once_release(mi_atomic_once_t* once) { } } +#if MI_USE_PTHREADS +mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init) { + int err = pthread_key_create(pkey,destruct); + if mi_unlikely(err!=0) { + *pkey = MI_PTHREAD_KEY_INVALID; + _mi_error_message(ENOMEM,"unable to allocate a thread local variable (error %d)\n", err); + return false; + } + mi_assert_internal(*pkey != MI_PTHREAD_KEY_INVALID); + if (init!=NULL) { + pthread_setspecific(*pkey,init); + }; + mi_assert_internal(pthread_getspecific(*pkey)==init); + return true; +} +#endif + +// -------------------------------------------------------- +// Detect CPU features +// -------------------------------------------------------- +mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast +mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast +mi_decl_cache_align bool _mi_cpu_has_popcnt = false; + +#if (MI_ARCH_X64 || MI_ARCH_X86) +#if defined(__GNUC__) +// #include +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx) + // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect). + uint32_t eax, ebx, ecx, edx; + __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : ); + regs4[0] = eax; + regs4[1] = ebx; + regs4[2] = ecx; + regs4[3] = edx; + return true; +} + +#elif defined(_MSC_VER) +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel); + return true; +} +#else +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel); + return false; +} +#endif + +void _mi_detect_cpu_features(void) { + // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) + // FSRS for fast short rep stosb + bool amd = false; + bool fsrm = false; + // bool erms = false; + bool fsrs = false; + uint32_t cpu_info[4]; + if (mi_cpuid(cpu_info, 0, 0)) { + amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) + } + if (mi_cpuid(cpu_info, 7, 0)) { + fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see + // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see + } + if (mi_cpuid(cpu_info, 7, 1)) { + fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see + } + if (mi_cpuid(cpu_info, 1, 0)) { + _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see + } + + if (fsrm) { + _mi_cpu_movsb_max = 127; + } + if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 + _mi_cpu_stosb_max = 127; + } +} + +#else +void _mi_detect_cpu_features(void) { + #if MI_ARCH_ARM64 + _mi_cpu_has_popcnt = true; + #endif +} +#endif + // -------------------------------------------------------- // Define our own limited `_mi_vsnprintf` and `_mi_snprintf` @@ -198,22 +289,20 @@ static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* mi_outc('0',out,end); } else { - // output digits in reverse - char* start = *out; - while (x > 0) { + #define MI_MAX_OUT_DIGITS (160) /* a 512 bit number has 155 digits */ + char num[MI_MAX_OUT_DIGITS]; + int dcount = 0; + while(x>0 && dcount < MI_MAX_OUT_DIGITS) { char digit = (char)(x % base); - mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end); + num[dcount++] = (digit <= 9 ? '0' + digit : 'A' + digit - 10); x = x / base; } + if (dcount>=MI_MAX_OUT_DIGITS) return; // don't output anything? if (prefix != 0) { mi_outc(prefix, out, end); } - size_t len = *out - start; - // and reverse in-place - for (size_t i = 0; i < (len / 2); i++) { - char c = start[len - i - 1]; - start[len - i - 1] = start[i]; - start[i] = c; + while(dcount-- > 0) { + mi_outc(num[dcount], out, end); } } } @@ -260,7 +349,10 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { if (c >= '1' && c <= '9') { width = (c - '0'); MI_NEXTC(); while (c >= '0' && c <= '9') { - width = (10 * width) + (c - '0'); MI_NEXTC(); + if (width < SIZE_MAX/1024) { // no overflow + width = (10 * width) + (c - '0'); + } + MI_NEXTC(); } if (c == 0) break; // extra check due to while } @@ -299,7 +391,7 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { if (width == 0 && (c == 'x' || c == 'p')) { if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); } if (width == 0) { width = 2; } - fill = '0'; + if (alignright) { fill = '0'; } } mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end); } @@ -352,6 +444,7 @@ int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) { return written; } +#undef MI_NEXTC // -------------------------------------------------------- @@ -367,7 +460,7 @@ static size_t mi_ctz_generic32(uint32_t x) { 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; if (x==0) return 32; - return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27]; + return debruijn[(uint32_t)((x & (~x + 1U)) * (uint32_t)(0x077CB531U)) >> 27]; } static size_t mi_clz_generic32(uint32_t x) { diff --git a/system/lib/mimalloc/src/options.c b/system/lib/mimalloc/src/options.c index 2873f3c38ec00..da0d6669c48c8 100644 --- a/system/lib/mimalloc/src/options.c +++ b/system/lib/mimalloc/src/options.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -17,7 +17,7 @@ static long mi_max_warning_count = 16; // stop outputting warnings after this (u static void mi_add_stderr_output(void); -int mi_version(void) mi_attr_noexcept { +int mi_version(void) { return MI_MALLOC_VERSION; } @@ -85,7 +85,7 @@ int mi_version(void) mi_attr_noexcept { #endif #ifndef MI_DEFAULT_PAGEMAP_COMMIT -#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access +#if defined(__APPLE__) && MI_PAGE_MAP_FLAT // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access #define MI_DEFAULT_PAGEMAP_COMMIT 1 #else #define MI_DEFAULT_PAGEMAP_COMMIT 0 @@ -104,7 +104,8 @@ int mi_version(void) mi_attr_noexcept { #if defined(__ANDROID__) #define MI_DEFAULT_ALLOW_THP 0 #else -#define MI_DEFAULT_ALLOW_THP 1 +// #define MI_DEFAULT_ALLOW_THP 1 // allow THP but purging may split up THP pages +#define MI_DEFAULT_ALLOW_THP 2 // allow THP and set the minimal purge size to 2MiB to avoid breaking them up #endif #endif @@ -146,15 +147,11 @@ static mi_option_desc_t mi_options[_mi_option_last] = { 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { 4, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) }, { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. -#if defined(MI_VISIT_ABANDONED) - { 1, MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim. -#else - { 0, MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) }, -#endif + { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_visit_abandoned) }, { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) @@ -175,7 +172,7 @@ static mi_option_desc_t mi_options[_mi_option_last] = MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class { MI_DEFAULT_ALLOW_THP, MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. - { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`). + { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation @@ -221,7 +218,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ const int vermajor = MI_MALLOC_VERSION/10000; const int verminor = (MI_MALLOC_VERSION%10000)/100; const int verpatch = (MI_MALLOC_VERSION%100); - _mi_fprintf(out, arg, "v%i.%i.%i%s%s (built on %s, %s)\n", vermajor, verminor, verpatch, + _mi_fprintf(out, arg, "v%i.%i.%i%s%s\n", vermajor, verminor, verpatch, #if defined(MI_CMAKE_BUILD_TYPE) ", " mi_stringify(MI_CMAKE_BUILD_TYPE) #else @@ -233,7 +230,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #else "" #endif - , __DATE__, __TIME__); + ); // show options for (int i = 0; i < _mi_option_last; i++) { @@ -253,6 +250,20 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #if MI_TSAN _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif + #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED + _mi_fprintf(out, arg, "free: (small) aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_PAGE_META_IS_ALIGNED + _mi_fprintf(out, arg, "free: aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_PAGE_META_SMALL_IS_ALIGNED + _mi_fprintf(out, arg, "free: small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_FREE_IS_CHECKED + _mi_fprintf(out, arg, "free: checked, page size: %zu\n", sizeof(mi_page_t)); + #else + _mi_fprintf(out, arg, "free: pagemap, page size: %zu\n", sizeof(mi_page_t)); + #endif + #if MI_ENCODE_FREELIST + _mi_fprintf(out, arg, "free lists: encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + #endif } mi_decl_export void mi_options_print(void) mi_attr_noexcept { @@ -437,25 +448,12 @@ static void mi_add_stderr_output(void) { static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings -// When overriding malloc, we may recurse into mi_vfprintf if an allocation -// inside the C runtime causes another message. -// In some cases (like on macOS) the loader already allocates which -// calls into mimalloc; if we then access thread locals (like `recurse`) -// this may crash as the access may call _tlv_bootstrap that tries to -// (recursively) invoke malloc again to allocate space for the thread local -// variables on demand. This is why we use a _mi_preloading test on such -// platforms. However, C code generator may move the initial thread local address -// load before the `if` and we therefore split it out in a separate function. -static mi_decl_thread bool recurse = false; - static mi_decl_noinline bool mi_recurse_enter_prim(void) { - if (recurse) return false; - recurse = true; return true; } static mi_decl_noinline void mi_recurse_exit_prim(void) { - recurse = false; + /* nothing */ } static bool mi_recurse_enter(void) { @@ -473,7 +471,7 @@ static void mi_recurse_exit(void) { } void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) { - if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr? + if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // todo: use mi_out_stderr for stderr? if (!mi_recurse_enter()) return; out = mi_out_get_default(&arg); if (prefix != NULL) out(prefix, arg); @@ -505,7 +503,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { } static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { - if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) { + if (prefix != NULL && _mi_strnlen(prefix,33) <= 32) { // && !_mi_is_main_thread()) { char tprefix[64]; _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id()); mi_vfprintf(out, arg, tprefix, fmt, args); @@ -522,13 +520,6 @@ void _mi_raw_message(const char* fmt, ...) { va_end(args); } -void _mi_message(const char* fmt, ...) { - va_list args; - va_start(args, fmt); - mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args); - va_end(args); -} - void _mi_trace_message(const char* fmt, ...) { if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher va_list args; @@ -581,24 +572,27 @@ static _Atomic(void*) mi_error_arg; // = NULL static void mi_error_default(int err) { MI_UNUSED(err); -#if (MI_DEBUG>0) - if (err==EFAULT) { - #ifdef _MSC_VER - __debugbreak(); - #endif - abort(); - } -#endif -#if (MI_SECURE>0) - if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) - abort(); - } -#endif -#if defined(MI_XMALLOC) - if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode - abort(); + #if (MI_DEBUG>0) + if (err==EFAULT) { + #ifdef _MSC_VER + __debugbreak(); + #endif + abort(); + } + #endif + #if (MI_SECURE>0) + if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) + abort(); + } + #endif + #if defined(MI_XMALLOC) + if (err==ENOMEM || err==EOVERFLOW || err==EINVAL) { // abort on memory allocation fails in xmalloc mode + abort(); + } + #endif + if (errno==0) { + errno = (err==EINVAL ? EINVAL : ENOMEM /* compatibility */ ); } -#endif } void mi_register_error(mi_error_fun* fun, void* arg) { @@ -612,7 +606,7 @@ void _mi_error_message(int err, const char* fmt, ...) { va_start(args, fmt); mi_show_error_message(fmt, args); va_end(args); - // and call the error handler which may abort (or return normally) + // and call the error handler which may abort (or return normally, potentially setting errno) if (mi_error_handler != NULL) { mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg)); } @@ -621,14 +615,17 @@ void _mi_error_message(int err, const char* fmt, ...) { } } +mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) { + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + return NULL; +} + // -------------------------------------------------------- // Initialize options by checking the environment // -------------------------------------------------------- // TODO: implement ourselves to reduce dependencies on the C runtime #include // strtol -#include // strstr - static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment @@ -662,8 +659,9 @@ static void mi_option_init(mi_option_desc_t* desc) { } else { char* end = buf; + errno = 0; long value = strtol(buf, &end, 10); - if (mi_option_has_size_in_kib(desc->option)) { + if (errno==0 && mi_option_has_size_in_kib(desc->option)) { // this option is interpreted in KiB to prevent overflow of `long` for large allocations // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) size_t size = (value < 0 ? 0 : (size_t)value); @@ -678,7 +676,7 @@ static void mi_option_init(mi_option_desc_t* desc) { if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); } value = (size > LONG_MAX ? LONG_MAX : (long)size); } - if (*end == 0) { + if (errno==0 && *end == 0) { mi_option_set(desc->option, value); } else { diff --git a/system/lib/mimalloc/src/os.c b/system/lib/mimalloc/src/os.c index 5edcca1b68642..a727cc844a715 100644 --- a/system/lib/mimalloc/src/os.c +++ b/system/lib/mimalloc/src/os.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -8,6 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "mimalloc/atomic.h" #include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" // _mi_theap_default for random /* ----------------------------------------------------------- Initialization. @@ -40,6 +41,9 @@ bool _mi_os_has_virtual_reserve(void) { return mi_os_mem_config.has_virtual_reserve; } +bool _mi_os_canuse_thp(void) { + return mi_os_mem_config.has_transparent_huge_pages; +} // OS (small) page size size_t _mi_os_page_size(void) { @@ -54,13 +58,19 @@ size_t _mi_os_large_page_size(void) { // minimal purge size. Can be larger than the page size if transparent huge pages are enabled. size_t _mi_os_minimal_purge_size(void) { size_t minsize = mi_option_get_size(mi_option_minimal_purge_size); - if (minsize != 0) { + if (minsize != 0) { + // set by user return _mi_align_up(minsize, _mi_os_page_size()); } else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { + // don't break up THP pages; + // we don't do this by default as that can lead to increased memory usage (see issue #1282). + // on the other hand, breaking up THP pages can lead to performance degradation and perhaps + // allow_thp should be 2 by default (and the user can set the OS THP setting to [never]). return _mi_os_large_page_size(); } else { + // OS page size by default return _mi_os_page_size(); } } @@ -103,55 +113,57 @@ void _mi_os_init(void) { /* ----------------------------------------------------------- Util -------------------------------------------------------------- */ -bool _mi_os_decommit(void* addr, size_t size); -bool _mi_os_commit(void* addr, size_t size, bool* is_zero); +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); +bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero); // On systems with enough virtual address bits, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address // space (64TiB) we use this technique. (but see issue #939) #if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) -// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available. +// Return a `try_alignment` aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be // properly aligned which can be more costly as it needs to be adjusted afterwards. // In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization; // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) -#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment #define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start #define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) #define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) -void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) { static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 // todo: perhaps only do alignment hints if THP is enabled? - if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL; + if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space - size = _mi_align_up(size, MI_HINT_ALIGN); + + size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. + req_size += (try_alignment - 1); // ensure we can align in the requested size + req_size = _mi_align_up(req_size, _mi_os_large_page_size()); #if (MI_SECURE>=1) - if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 + if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 #endif - size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. - - uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); + + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode + #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); - init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB #endif - uintptr_t expected = hint + size; + uintptr_t expected = hint + req_size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + if (hint==0) return NULL; } - mi_assert_internal(hint%MI_HINT_ALIGN == 0); - if (hint%try_alignment != 0) return NULL; - return (void*)hint; + const uintptr_t hint_align = _mi_align_up(hint,try_alignment); + mi_assert_internal(hint_align + sze < hint + req_size); + return (void*)hint_align; } #else void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { @@ -175,7 +187,7 @@ size_t _mi_os_secure_guard_page_size(void) { } // In secure mode, try to decommit an area and output a warning if this fails. -bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) { +bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { if (addr == NULL) return true; #if MI_SECURE > 0 bool ok = false; @@ -185,7 +197,7 @@ bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) { ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg); } else { - ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size()); + ok = _mi_os_decommit(subproc, addr, _mi_os_secure_guard_page_size()); } } if (!ok) { @@ -193,18 +205,18 @@ bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) { } return ok; #else - MI_UNUSED(memid); + MI_UNUSED(subproc); MI_UNUSED(memid); return true; #endif } // In secure mode, try to decommit an area and output a warning if this fails. -bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) { - return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); +bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); } // In secure mode, try to recommit an area -bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) { +bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { if (addr == NULL) return true; #if MI_SECURE > 0 if (!memid.is_pinned) { @@ -213,18 +225,18 @@ bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) { return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg); } else { - return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL); + return _mi_os_commit(subproc, addr, _mi_os_secure_guard_page_size(), NULL); } } #else - MI_UNUSED(memid); + MI_UNUSED(subproc); MI_UNUSED(memid); #endif return true; } // In secure mode, try to recommit an area -bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) { - return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); +bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + return _mi_os_secure_guard_page_reset_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); } @@ -232,23 +244,27 @@ bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) { Free memory -------------------------------------------------------------- */ -static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc); +static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size); -static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) { +static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) { + mi_assert_internal(subproc!=NULL); mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr) int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041) if (err != 0) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } - if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer - if (commit_size > 0) { - mi_subproc_stat_decrease(subproc, committed, commit_size); + if (adjust) { + if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); } + mi_subproc_stat_adjust_decrease(subproc, reserved, size); + } + else { + if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); } + mi_subproc_stat_decrease(subproc, reserved, size); } - mi_subproc_stat_decrease(subproc, reserved, size); } -void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) { +void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) { if (mi_memkind_is_os(memid.memkind)) { size_t csize = memid.mem.os.size; if (csize==0) { csize = _mi_os_good_alloc_size(size); } @@ -270,10 +286,10 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me // free it if (memid.memkind == MI_MEM_OS_HUGE) { mi_assert(memid.is_pinned); - mi_os_free_huge_os_pages(base, csize, subproc); + mi_os_free_huge_os_pages(subproc, base, csize); } else { - mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc); + mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false); } } else { @@ -282,8 +298,8 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me } } -void _mi_os_free(void* p, size_t size, mi_memid_t memid) { - _mi_os_free_ex(p, size, true, memid, NULL); +void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { + _mi_os_free_ex(subproc, p, size, true, memid); } @@ -293,7 +309,7 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid) { // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. // Also `hint_addr` is a hint and may be ignored. -static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { +static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(is_zero != NULL); mi_assert_internal(is_large != NULL); @@ -314,11 +330,11 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large); } - mi_os_stat_counter_increase(mmap_calls, 1); + mi_subproc_stat_counter_increase(subproc, mmap_calls, 1); if (p != NULL) { - mi_os_stat_increase(reserved, size); + mi_subproc_stat_increase(subproc, reserved, size); if (commit) { - mi_os_stat_increase(committed, size); + mi_subproc_stat_increase(subproc, committed, size); // seems needed for asan (or `mimalloc-test-api` fails) #ifdef MI_TRACK_ASAN if (*is_zero) { mi_track_mem_defined(p,size); } @@ -329,14 +345,14 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm return p; } -static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { - return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero); +static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { + return mi_os_prim_alloc_at(subproc, NULL /* hint addr */, size, try_alignment, commit, allow_large, is_large, is_zero); } // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. -static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) { +static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) { mi_assert_internal(memid!=NULL); mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); @@ -345,8 +361,12 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); + #if MI_INTPTR_SIZE >= 8 + const bool try_direct_alloc = true; + #else // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4); + #endif bool os_is_large = false; bool os_is_zero = false; @@ -354,7 +374,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit size_t os_size = size; void* p = NULL; if (try_direct_alloc) { - p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero); + p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero); } // aligned already? @@ -368,33 +388,34 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); } #endif - if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); } + if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); } if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block // over-allocate uncommitted (virtual) memory - p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero); + p = mi_os_prim_alloc(subproc, over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero); if (p == NULL) return NULL; // set p to the aligned part in the full region // note: Windows VirtualFree needs the actual base pointer // this is handled though by having the `base` field in the memid os_base = p; // remember the base - os_size = over_size; + os_size = over_size; // todo: use size instead as now we over-decrement commit stats on free? p = _mi_align_up_ptr(p, alignment); // explicitly commit only the aligned part if (commit) { - if (!_mi_os_commit(p, size, NULL)) { - mi_os_prim_free(os_base, over_size, 0, NULL); + if (!_mi_os_commit(subproc, p, size, NULL)) { + mi_os_prim_free(subproc, os_base, over_size, 0, true); return NULL; } } } else { // mmap can free inside an allocation // overallocate... - p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero); + // note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though. + p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero); if (p == NULL) return NULL; // and selectively unmap parts around the over-allocated area. @@ -403,8 +424,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit const size_t mid_size = _mi_align_up(size, _mi_os_page_size()); const size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); } - if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); } + if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); } + if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); } // we can return the aligned pointer on `mmap` systems p = aligned_p; os_base = aligned_p; // since we freed the pre part, `*base == p`. @@ -423,13 +444,13 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit OS API: alloc and alloc_aligned ----------------------------------------------------------- */ -void* _mi_os_alloc(size_t size, mi_memid_t* memid) { +void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) { *memid = _mi_memid_none(); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); bool os_is_large = false; bool os_is_zero = false; - void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero); + void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero); if (p == NULL) return NULL; *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); @@ -438,7 +459,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid) { return p; } -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) +void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) { MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings *memid = _mi_memid_none(); @@ -446,7 +467,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); - void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid ); + void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid ); if (p == NULL) return NULL; mi_assert_internal(memid->mem.os.size >= size); @@ -456,13 +477,13 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo } -mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) { +mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) { if (p==NULL || size==0) return p; // ensure committed if (!memid->initially_committed) { bool is_zero = false; - if (!_mi_os_commit(p, size, &is_zero)) { - _mi_os_free(p, size, *memid); + if (!_mi_os_commit(subproc, p, size, &is_zero)) { + _mi_os_free(subproc, p, size, *memid); return NULL; } memid->initially_committed = true; @@ -474,9 +495,9 @@ mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_ return p; } -void* _mi_os_zalloc(size_t size, mi_memid_t* memid) { - void* p = _mi_os_alloc(size,memid); - return mi_os_ensure_zero(p, size, memid); +void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) { + void* p = _mi_os_alloc(subproc, size,memid); + return mi_os_ensure_zero(subproc, p, size, memid); } /* ----------------------------------------------------------- @@ -487,28 +508,29 @@ void* _mi_os_zalloc(size_t size, mi_memid_t* memid) { to use the actual start of the memory region. ----------------------------------------------------------- */ -void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) { +void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) { mi_assert(offset <= size); mi_assert((alignment % _mi_os_page_size()) == 0); *memid = _mi_memid_none(); if (offset > size) return NULL; if (offset == 0) { // regular aligned allocation - return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid); + return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid); } else { // overallocate to align at an offset const size_t extra = _mi_align_up(offset, alignment) - offset; if (size >= SIZE_MAX - extra) return NULL; // too large const size_t oversize = size + extra; - void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid); + void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid); if (start == NULL) return NULL; void* const p = (uint8_t*)start + extra; mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); // decommit the overallocation at the start + // note: this double counts the decommit when freeing `memid`. Should we keep commit size in the memid as well? if (commit && extra >= _mi_os_page_size()) { - _mi_os_decommit(start, extra); + _mi_os_decommit(subproc, start, extra); } return p; } @@ -542,9 +564,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) { +bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(size >= stat_already_committed); if (is_zero != NULL) { *is_zero = false; } - mi_os_stat_counter_increase(commit_calls, 1); + mi_subproc_stat_counter_increase(subproc, commit_calls, 1); // page align range size_t csize; @@ -567,15 +590,15 @@ bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) if (os_is_zero) { mi_track_mem_defined(start,csize); } else { mi_track_mem_undefined(start,csize); } #endif - mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit + mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommit return true; } -bool _mi_os_commit(void* addr, size_t size, bool* is_zero) { - return _mi_os_commit_ex(addr, size, is_zero, size); +bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) { + return _mi_os_commit_ex(subproc, addr, size, is_zero, 0); } -static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) { +static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) { mi_assert_internal(needs_recommit!=NULL); // page align @@ -590,15 +613,15 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, siz _mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); } else if (*needs_recommit) { - mi_os_stat_decrease(committed, stat_size); + mi_subproc_stat_decrease(subproc, committed, stat_size); } mi_assert_internal(err == 0); return (err == 0); } -bool _mi_os_decommit(void* addr, size_t size) { +bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) { bool needs_recommit; - return mi_os_decommit_ex(addr, size, &needs_recommit, size); + return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size); } @@ -606,13 +629,13 @@ bool _mi_os_decommit(void* addr, size_t size) { // but may be used later again. This will release physical memory // pages and reduce swapping while keeping the memory committed. // We page align to a conservative area inside the range to reset. -bool _mi_os_reset(void* addr, size_t size) { +bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) { // page align conservatively within the range size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) - mi_os_stat_counter_increase(reset, csize); - mi_os_stat_counter_increase(reset_calls, 1); + mi_subproc_stat_counter_increase(subproc, reset, csize); + mi_subproc_stat_counter_increase(subproc, reset_calls, 1); #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN memset(start, 0, csize); // pretend it is eagerly reset @@ -626,7 +649,8 @@ bool _mi_os_reset(void* addr, size_t size) { } -void _mi_os_reuse( void* addr, size_t size ) { +void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) { + MI_UNUSED(subproc); // page align conservatively within the range size_t csize = 0; void* const start = mi_os_page_align_area_conservative(addr, size, &csize); @@ -639,11 +663,11 @@ void _mi_os_reuse( void* addr, size_t size ) { // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. -bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg) +bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg) { if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? - mi_os_stat_counter_increase(purge_calls, 1); - mi_os_stat_counter_increase(purged, size); + mi_subproc_stat_counter_increase(subproc, purge_calls, 1); + mi_subproc_stat_counter_increase(subproc, purged, size); if (commit_fun != NULL) { bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg); @@ -653,12 +677,12 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m !_mi_preloading()) // don't decommit during preloading (unsafe) { bool needs_recommit = true; - mi_os_decommit_ex(p, size, &needs_recommit, stat_size); + mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size); return needs_recommit; } else { if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory) - _mi_os_reset(p, size); + _mi_os_reset(subproc, p, size); } return false; // needs no recommit } @@ -666,8 +690,8 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. -bool _mi_os_purge(void* p, size_t size) { - return _mi_os_purge_ex(p, size, true, size, NULL, NULL); +bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) { + return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL); } @@ -713,7 +737,11 @@ static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 // Claim an aligned address range for huge pages static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { if (total_size != NULL) *total_size = 0; - const size_t size = pages * MI_HUGE_OS_PAGE_SIZE; + size_t size = 0; + if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size)) { + _mi_warning_message("too many huge pages requested: %zu\n", pages); + return NULL; + } uintptr_t start = 0; uintptr_t end = 0; @@ -722,7 +750,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { start = huge_start; if (start == 0) { // Initialize the start address after the 32TiB area - start = ((uintptr_t)8 << 40); // 8TiB virtual start address + start = ((uintptr_t)32 << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint) #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness? @@ -749,7 +777,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { #endif // Allocate MI_ARENA_SLICE_ALIGN aligned huge pages -void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) { +void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) { *memid = _mi_memid_none(); if (psize != NULL) *psize = 0; if (pages_reserved != NULL) *pages_reserved = 0; @@ -780,21 +808,21 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // no success, issue a warning and break if (p != NULL) { _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); - mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL); + mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */); } break; } // success, record it page++; // increase before timeout check (see issue #711) - mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE); - mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE); + mi_subproc_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE); + mi_subproc_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE); // check for timeout if (max_msecs > 0) { mi_msecs_t elapsed = _mi_clock_end(start_t); if (page >= 1) { - mi_msecs_t estimate = ((elapsed / (page+1)) * pages); + mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages); if (estimate > 2*max_msecs) { // seems like we are going to timeout, break elapsed = max_msecs + 1; } @@ -823,11 +851,11 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // free every huge page in a range individually (as we allocated per page) // note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. -static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) { +static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) { if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { - mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc); + mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false /* adjust? */); size -= MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE; } diff --git a/system/lib/mimalloc/src/page-map.c b/system/lib/mimalloc/src/page-map.c index b213eb098ce83..949e37ac3afab 100644 --- a/system/lib/mimalloc/src/page-map.c +++ b/system/lib/mimalloc/src/page-map.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2023-2025, Microsoft Research, Daan Leijen +Copyright (c) 2023-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -25,8 +25,11 @@ static void mi_page_map_cannot_commit(void) { // A full 256 TiB address space (48 bit) needs a 4 GiB page map. // A full 4 GiB address space (32 bit) needs only a 64 KiB page map. -mi_decl_cache_align uint8_t* _mi_page_map = NULL; -static void* mi_page_map_max_address = NULL; +// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) +static uint8_t mi_page_map_empty[1] = { 1 }; // _mi_ptr_page(NULL) == NULL + +mi_decl_hidden mi_decl_cache_align _Atomic(uint8_t*) _mi_page_map = mi_page_map_empty; +mi_decl_hidden _Atomic(void*) _mi_page_map_max_address = NULL; static mi_memid_t mi_page_map_memid; #define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE @@ -35,7 +38,7 @@ static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count); bool _mi_page_map_init(void) { - size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS); + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); if (vbits == 0) { vbits = _mi_os_virtual_address_bits(); #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB @@ -45,15 +48,22 @@ bool _mi_page_map_init(void) { if (vbits < MI_ARENA_SLICE_SHIFT) { vbits = MI_ARENA_SLICE_SHIFT; } + if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr + vbits = MI_MIN_VABITS; + } + if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available + vbits = MI_MAX_VABITS; + } // Allocate the page map and commit bits - mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits)); + mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits))); const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT)); const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems? const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL)); const size_t reserve_size = bitmap_size + page_map_size; - uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); + mi_subproc_t* const subproc = _mi_subproc_main(); + uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); if (base==NULL) { _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB); return false; @@ -64,13 +74,13 @@ bool _mi_page_map_init(void) { } if (bitmap_size > 0) { mi_page_map_commit = (mi_bitmap_t*)base; - if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) { + if (!_mi_os_commit(subproc, mi_page_map_commit, bitmap_size, NULL)) { mi_page_map_cannot_commit(); return false; } mi_bitmap_init(mi_page_map_commit, commit_bits, true); } - _mi_page_map = base + bitmap_size; + mi_atomic_store_ptr_release(uint8_t,&_mi_page_map, base + bitmap_size); // commit the first part so NULL pointers get resolved without an access violation if (!commit) { @@ -79,19 +89,18 @@ bool _mi_page_map_init(void) { return false; } } - _mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL + mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so _mi_ptr_page(NULL) == NULL mi_assert_internal(_mi_ptr_page(NULL)==NULL); return true; } -void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) { - mi_assert_internal(subproc != NULL); - mi_assert_internal(_mi_page_map != NULL); - if (_mi_page_map == NULL) return; - _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc); - _mi_page_map = NULL; +void _mi_page_map_unsafe_destroy(void) { + mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) != NULL); + if (mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) == NULL) return; + _mi_os_free_ex(_mi_subproc_main(), mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid); + mi_atomic_store_ptr_release(uint8_t, &_mi_page_map, NULL); mi_page_map_commit = NULL; - mi_page_map_max_address = NULL; + mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, NULL); mi_page_map_memid = _mi_memid_none(); } @@ -100,6 +109,7 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) { // is the page map area that contains the page address committed? // we always set the commit bits so we can track what ranges are in-use. // we only actually commit if the map wasn't committed fully already. + uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); if (mi_page_map_commit != NULL) { const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; @@ -107,9 +117,9 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) { if (mi_bitmap_is_clear(mi_page_map_commit, i)) { // this may race, in which case we do multiple commits (which is ok) bool is_zero; - uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); + uint8_t* const start = page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; - if (!_mi_os_commit(start, size, &is_zero)) { + if (!_mi_os_commit(_mi_subproc_main(), start, size, &is_zero)) { mi_page_map_cannot_commit(); return false; } @@ -119,8 +129,8 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) { } } #if MI_DEBUG > 0 - _mi_page_map[idx] = 0; - _mi_page_map[idx+slice_count-1] = 0; + page_map[idx] = 0; + page_map[idx+slice_count-1] = 0; #endif return true; } @@ -137,11 +147,13 @@ static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* bool _mi_page_map_register(mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access! - if mi_unlikely(_mi_page_map == NULL) { + mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) != NULL); // should be initialized before multi-thread access! + uint8_t* page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + if mi_unlikely(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) == NULL) { if (!_mi_page_map_init()) return false; + page_map = mi_atomic_load_ptr_acquire(uint8_t,&_mi_page_map); } - mi_assert(_mi_page_map!=NULL); + mi_assert(page_map!=NULL); uint8_t* page_start; size_t slice_count; const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); @@ -153,37 +165,42 @@ bool _mi_page_map_register(mi_page_t* page) { // set the offsets for (size_t i = 0; i < slice_count; i++) { mi_assert_internal(i < 128); - _mi_page_map[idx + i] = (uint8_t)(i+1); + page_map[idx + i] = (uint8_t)(i+1); } return true; } void _mi_page_map_unregister(mi_page_t* page) { - mi_assert_internal(_mi_page_map != NULL); + uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + mi_assert_internal(page_map != NULL); + if (page_map == NULL) return; // get index and count uint8_t* page_start; size_t slice_count; const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); // unset the offsets - _mi_memzero(_mi_page_map + idx, slice_count); + _mi_memzero(page_map + idx, slice_count); } void _mi_page_map_unregister_range(void* start, size_t size) { + uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); + mi_assert_internal(page_map!=NULL); + if (page_map == NULL) return; const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); const uintptr_t index = _mi_page_map_index(start); // todo: scan the commit bits and clear only those ranges? if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total; return; } - _mi_memzero(&_mi_page_map[index], slice_count); + _mi_memzero(&page_map[index], slice_count); } mi_page_t* _mi_safe_ptr_page(const void* p) { - if mi_unlikely(p >= mi_page_map_max_address) return NULL; + if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; const uintptr_t idx = _mi_page_map_index(p); if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL; - const uintptr_t ofs = _mi_page_map[idx]; + const uintptr_t ofs = _mi_page_map_at(idx); if mi_unlikely(ofs == 0) return NULL; return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT); } @@ -196,43 +213,64 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att // A 2-level page map #define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) -#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS)) -mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map; -static size_t mi_page_map_count; -static void* mi_page_map_max_address; -static mi_memid_t mi_page_map_memid; -static mi_lock_t mi_page_map_lock; +// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) +static mi_page_map_t mi_page_map_empty = { + MI_ATOMIC_VAR_INIT(1), + sizeof(mi_page_map_t), + MI_MEMID_STATIC, + MI_LOCK_INITIALIZER, + { MI_ATOMIC_VAR_INIT(NULL) } +}; + +mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty); -// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand -static _Atomic(mi_bfield_t) mi_page_map_commit; +static size_t mi_page_map_count_of_size(size_t size) { + return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_submap_t)); +} + +// static void* mi_page_map_addr_of_index(size_t idx) { +// return (void*)((uintptr_t)idx * MI_PAGE_MAP_SUB_COUNT * MI_ARENA_SLICE_SIZE); +// } -mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) { - mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit); - const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT; - mi_assert_internal(bit_idx < MI_BFIELD_BITS); - if (pbit_idx != NULL) { *pbit_idx = bit_idx; } - return ((commit & (MI_ZU(1) << bit_idx)) != 0); +mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) { + const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size); + if mi_unlikely(required_idx >= reserved_count) { + mi_page_map_cannot_commit(); + return false; + } + size_t commit_size = _mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE ); + if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; } + const size_t commit_count = mi_page_map_count_of_size(commit_size); + mi_assert_internal(commit_count > required_idx); + mi_assert_internal(commit_count <= reserved_count); + // note: we rely on uncommitted memory to be zero initialized on the first commit (and further concurrent commits leave the memory as is). + bool is_zero; + if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), pmap, commit_size, &is_zero)) { + mi_page_map_cannot_commit(); + return false; + } + mi_assert_internal(is_zero || pmap->memid.initially_zero); + mi_atomic_store_release(&pmap->committed_count, commit_count); + // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + return true; } -mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) { +mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { mi_assert_internal(submap!=NULL && *submap==NULL); - size_t bit_idx; - if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) { - uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT]; - if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) { - mi_page_map_cannot_commit(); - return false; + if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) { + if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) { + if (!mi_page_map_commit_entries(pmap,idx)) return false; + mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); } - mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx); } - *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx); + *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); return true; } -// initialize the page map -bool _mi_page_map_init(void) { - size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS); +// initialize the page map +static bool mi_page_map_init_once(void) { + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); if (vbits == 0) { vbits = _mi_os_virtual_address_bits(); #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB @@ -242,111 +280,145 @@ bool _mi_page_map_init(void) { if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) { vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT; } + if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr + vbits = MI_MIN_VABITS; + } + if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available + vbits = MI_MAX_VABITS; + } // Allocate the page map and commit bits mi_assert(MI_MAX_VABITS >= vbits); - mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits)); - mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); - mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT); - const size_t os_page_size = _mi_os_page_size(); - const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size); - const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; - const size_t reserve_size = page_map_size + submap_size; - #if MI_SECURE - const bool commit = true; // the whole page map is valid and we can reliably check any pointer - #else - const bool commit = page_map_size <= 64*MI_KiB || + mi_assert(MI_MIN_VABITS <= vbits); + const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t os_page_size = _mi_os_page_size(); + const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + const size_t extra_reserve_size = reserve_size + submap_size; + const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); - #endif - _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); - if (_mi_page_map==NULL) { - _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB); + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_memid_t memid; + mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); + if mi_unlikely(pmap==NULL) { + _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB); return false; } - if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) { - _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); - _mi_memzero_aligned(_mi_page_map, page_map_size); + + // commit + size_t commit_count; + if (memid.initially_committed) { + if (!memid.initially_zero) { + _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); + _mi_memzero_aligned(pmap, extra_reserve_size); + memid.initially_zero = true; + } + commit_count = mi_page_map_count_of_size(reserve_size); } - mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0))); - + else { + // commit first entries up to MI_MIN_VABITS entries + const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size); + mi_assert_internal(min_commit_size <= reserve_size); + bool is_zero; + if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) { + mi_page_map_cannot_commit(); + _mi_os_free(subproc,pmap,extra_reserve_size,memid); + return false; + }; + mi_assert_internal(is_zero || memid.initially_zero); + commit_count = mi_page_map_count_of_size(min_commit_size); + mi_assert_internal(commit_count >= min_commit_count); + } + // ensure there is a submap for the NULL address - mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already - if (!mi_page_map_memid.initially_committed) { - if (!_mi_os_commit(sub0, submap_size, NULL)) { // commit full submap (issue #1087) + mi_page_t** const sub0 = (mi_submap_t)((uint8_t*)pmap + reserve_size); // we reserved a submap part at the end already + if (!memid.initially_committed) { + if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { // commit full submap (issue #1087) mi_page_map_cannot_commit(); + _mi_os_free(subproc,pmap,extra_reserve_size,memid); return false; } } - if (!mi_page_map_memid.initially_zero) { // initialize low addresses with NULL + if (!memid.initially_zero) { // initialize low addresses with NULL _mi_memzero_aligned(sub0, submap_size); } - mi_submap_t nullsub = NULL; - if (!mi_page_map_ensure_committed(0,&nullsub)) { - mi_page_map_cannot_commit(); - return false; - } - mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0); - mi_lock_init(&mi_page_map_lock); // initialize late in case the lock init causes allocation - + + // initialize the fields + pmap->memid = memid; + pmap->reserved_size = reserve_size; + mi_lock_init(&pmap->lock); + mi_atomic_store_release(&pmap->committed_count, commit_count); + // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap); mi_assert_internal(_mi_ptr_page(NULL)==NULL); return true; } +bool _mi_page_map_init(void) { + bool ok = true; + mi_atomic_do_once { + ok = mi_page_map_init_once(); + } + return ok; +} -void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) { - mi_assert_internal(subproc != NULL); - mi_assert_internal(_mi_page_map != NULL); - if (_mi_page_map == NULL) return; - mi_lock_done(&mi_page_map_lock); - for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) - // free all sub-maps - if (mi_page_map_is_committed(idx, NULL)) { - mi_submap_t sub = _mi_page_map_at(idx); - if (sub != NULL) { - mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); - _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc); - mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL); - } +void _mi_page_map_unsafe_destroy(void) { + mi_page_map_t* const pmap = _mi_page_map(); + mi_assert_internal(pmap != NULL); + if (pmap == NULL || pmap == &mi_page_map_empty) return; + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_lock_done(&pmap->lock); + for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) + // free all sub-maps + mi_submap_t sub = _mi_page_map_at(pmap,idx); + if (sub != NULL) { + mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); + _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[idx], NULL); } } - _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc); - _mi_page_map = NULL; - mi_page_map_count = 0; - mi_page_map_memid = _mi_memid_none(); - mi_page_map_max_address = NULL; - mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0); + _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty); } - -mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) { - mi_assert_internal(submap!=NULL && *submap==NULL); +mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) { + // sub map not yet allocated, alloc now mi_submap_t sub = NULL; - if (!mi_page_map_ensure_committed(idx, &sub)) { - return false; - } - if mi_unlikely(sub == NULL) { - // sub map not yet allocated, alloc now - mi_lock(&mi_page_map_lock) + mi_lock(&pmap->lock) + { + sub = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); // reload + if (sub==NULL) // not yet allocated by another thread? { - sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload - if (sub==NULL) // not yet allocated by another thread? - { - mi_memid_t memid; - const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; - sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid); - if (sub==NULL) { - _mi_warning_message("internal error: unable to extend the page map\n"); - } - else { - mi_submap_t expect = NULL; - if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) { - // another thread already allocated it.. free and continue - _mi_os_free(sub, submap_size, memid); - sub = expect; - } + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_memid_t memid; + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); + if (sub==NULL) { + _mi_warning_message("internal error: unable to extend the page map\n"); + } + else { + mi_submap_t expect = NULL; + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &pmap->submaps[idx], &expect, sub)) { + // another thread already allocated it.. free and continue + _mi_os_free(subproc, sub, submap_size, memid); + sub = expect; } } } + } + return sub; +} + +mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { + mi_assert_internal(submap!=NULL && *submap==NULL); + mi_submap_t sub = NULL; + if (!mi_page_map_ensure_committed(pmap, idx, &sub)) { + return false; + } + if mi_unlikely(sub==NULL) { + sub = mi_page_map_alloc_submap_at(pmap, idx); if (sub==NULL) return false; // unable to allocate the submap.. } mi_assert_internal(sub!=NULL); @@ -354,11 +426,11 @@ mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap return true; } -static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { +static bool mi_page_map_set_range_prim(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { // is the page map area that contains the page address committed? while (slice_count > 0) { mi_submap_t sub = NULL; - if (!mi_page_map_ensure_submap_at(idx, &sub)) { + if (!mi_page_map_ensure_submap_at(pmap, idx, &sub)) { return false; }; mi_assert_internal(sub!=NULL); @@ -374,11 +446,11 @@ static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_i return true; } -static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { - if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) { +static bool mi_page_map_set_range(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { + if mi_unlikely(!mi_page_map_set_range_prim(pmap, page,idx,sub_idx,slice_count)) { // failed to commit, call again to reset the page pointer if needed if (page!=NULL) { - mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count); + mi_page_map_set_range_prim(pmap,NULL,idx,sub_idx,slice_count); } return false; } @@ -396,49 +468,46 @@ static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slic bool _mi_page_map_register(mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access! - if mi_unlikely(_mi_page_map == NULL) { + mi_page_map_t* pmap = _mi_page_map(); + mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access! + if mi_unlikely(pmap == NULL) { if (!_mi_page_map_init()) return false; + pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&__mi_page_map); } - mi_assert(_mi_page_map!=NULL); + mi_assert(pmap!=NULL); size_t slice_count; size_t sub_idx; const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count); - return mi_page_map_set_range(page, idx, sub_idx, slice_count); + return mi_page_map_set_range(pmap, page, idx, sub_idx, slice_count); } void _mi_page_map_unregister(mi_page_t* page) { mi_assert_internal(_mi_page_map != NULL); mi_assert_internal(page != NULL); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_page_map_t* const pmap = _mi_page_map(); // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena.c`) - if mi_unlikely(_mi_page_map == NULL) return; + if mi_unlikely(pmap == NULL) return; // get index and count size_t slice_count; size_t sub_idx; const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count); // unset the offsets - mi_page_map_set_range(NULL, idx, sub_idx, slice_count); + mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); } void _mi_page_map_unregister_range(void* start, size_t size) { - if mi_unlikely(_mi_page_map == NULL) return; + mi_page_map_t* const pmap = _mi_page_map(); + if mi_unlikely(pmap == NULL) return; const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); size_t sub_idx; const uintptr_t idx = _mi_page_map_index(start, &sub_idx); - mi_page_map_set_range(NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? + mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? } // Return NULL for invalid pointers mi_page_t* _mi_safe_ptr_page(const void* p) { - if (p==NULL) return NULL; - if mi_unlikely(p >= mi_page_map_max_address) return NULL; - size_t sub_idx; - const size_t idx = _mi_page_map_index(p,&sub_idx); - if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL; - mi_page_t** const sub = _mi_page_map[idx]; - if mi_unlikely(sub==NULL) return NULL; - return sub[sub_idx]; + return _mi_checked_ptr_page(p); } mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { diff --git a/system/lib/mimalloc/src/page-queue.c b/system/lib/mimalloc/src/page-queue.c index 8467f7acee704..c10c0925704fd 100644 --- a/system/lib/mimalloc/src/page-queue.c +++ b/system/lib/mimalloc/src/page-queue.c @@ -61,7 +61,7 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { // Returns MI_BIN_HUGE if the size is too large. // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. -static mi_decl_noinline size_t mi_bin(size_t size) { +static size_t mi_bin(size_t size) { size_t wsize = _mi_wsize_from_size(size); #if defined(MI_ALIGN4W) if mi_likely(wsize <= 4) { @@ -212,7 +212,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ if (size > MI_SMALL_SIZE_MAX) return; mi_page_t* page = pq->first; - if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty; + if (pq->first == NULL) page = _mi_page_empty_get(); // find index in the right direct page array const size_t idx = _mi_wsize_from_size(size); @@ -421,38 +421,3 @@ static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`) mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page); } - -// Only called from `mi_theap_absorb`. -size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) { - mi_assert_internal(mi_theap_contains_queue(theap,pq)); - mi_assert_internal(pq->block_size == append->block_size); - - if (append->first==NULL) return 0; - - // set append pages to new theap and count - size_t count = 0; - for (mi_page_t* page = append->first; page != NULL; page = page->next) { - mi_page_set_theap(page, theap); - count++; - } - mi_assert_internal(count == append->count); - - if (pq->last==NULL) { - // take over afresh - mi_assert_internal(pq->first==NULL); - pq->first = append->first; - pq->last = append->last; - mi_theap_queue_first_update(theap, pq); - } - else { - // append to end - mi_assert_internal(pq->last!=NULL); - mi_assert_internal(append->first!=NULL); - pq->last->next = append->first; - append->first->prev = pq->last; - pq->last = append->last; - } - pq->count += append->count; - - return count; -} diff --git a/system/lib/mimalloc/src/page.c b/system/lib/mimalloc/src/page.c index 24dd45dfd1c11..02ca72044c6f2 100644 --- a/system/lib/mimalloc/src/page.c +++ b/system/lib/mimalloc/src/page.c @@ -15,6 +15,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "mimalloc/atomic.h" #include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" /* ----------------------------------------------------------- Definition of page queues for each block size @@ -84,6 +85,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_block_size(page) > 0); mi_assert_internal(page->used <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); + + mi_assert_internal(page->heap!=NULL); + mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); // const size_t bsize = mi_page_block_size(page); // uint8_t* start = mi_page_start(page); @@ -99,7 +104,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); } } - #endif + #endif #if !MI_TRACK_ENABLED && !MI_TSAN mi_block_t* tfree = mi_page_thread_free(page); @@ -123,6 +128,9 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif if (!mi_page_is_abandoned(page)) { //mi_assert_internal(!_mi_process_is_initialized); + mi_assert_internal(page->heap!=NULL); + mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); { mi_page_queue_t* pq = mi_page_queue_of(page); mi_assert_internal(mi_page_queue_contains(pq, page)); @@ -155,7 +163,7 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free) if mi_unlikely(count > max_count) { - _mi_error_message(EFAULT, "corrupted thread-free list\n"); + _mi_error_message(EFAULT, "corrupted thread-free list (possibly due to a cross-thread double free)\n"); return; // the thread-free items cannot be freed } // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list) @@ -193,8 +201,8 @@ static void mi_page_thread_free_collect(mi_page_t* page) } // returns `true` if after collection `mi_page_immediate_available` is true. -static bool mi_page_free_quick_collect(mi_page_t* page) { - if (page->free != NULL) return true; +static inline bool mi_page_free_quick_collect(mi_page_t* page) { + if mi_likely(page->free != NULL) return true; if (page->local_free == NULL) return false; // move local_free to free page->free = page->local_free; @@ -265,24 +273,6 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { Page fresh and retire ----------------------------------------------------------- */ -/* -// called from segments when reclaiming abandoned pages -void _mi_page_reclaim(mi_theap_t* theap, mi_page_t* page) { - // mi_page_set_theap(page, theap); - // _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after theap is set) - _mi_page_free_collect(page, false); // ensure used count is up to date - - mi_assert_expensive(mi_page_is_valid_init(page)); - // mi_assert_internal(mi_page_theap(page) == theap); - // mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE); - - // TODO: push on full queue immediately if it is full? - mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); - mi_page_queue_push(theap, pq, page); - mi_assert_expensive(_mi_page_is_valid(page)); -} -*/ - // called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) { @@ -309,7 +299,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { mi_page_set_theap(page, NULL); page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap _mi_arenas_page_abandon(page, theap); - _mi_arenas_collect(false, false, theap->tld); // allow purging + // _mi_arenas_collect(false, false, theap->tld); // allow purging } } @@ -331,7 +321,9 @@ static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, si if (!mi_page_immediate_available(page)) { if (mi_page_is_expandable(page)) { if (!mi_page_extend_free(theap, page)) { - return NULL; // cannot commit + // cannot commit + _mi_page_abandon(page,pq); + return NULL; }; } else { @@ -416,11 +408,11 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL); mi_page_set_theap(page,NULL); _mi_arenas_page_free(page, theap); - _mi_arenas_collect(false, false, theap->tld); // allow purging + // _mi_arenas_collect(false, false, theap->tld); // allow purging } -#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE -#define MI_RETIRE_CYCLES (16) +#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */ +#define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */ // Retire a page with no more used blocks // Important to not retire too quickly though as new @@ -446,12 +438,10 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_page_queue_t* pq = mi_page_queue_of(page); #if MI_RETIRE_CYCLES > 0 const size_t bsize = mi_page_block_size(page); - if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? - if (pq->last==page && pq->first==page) { // the only page in the queue? + if mi_likely( pq->count <= MI_RETIRE_MAX_PAGES && !mi_page_queue_is_special(pq)) { // not full or huge queue? + if (pq->count==1 || bsize < MI_SMALL_SIZE_MAX) { mi_theap_t* theap = mi_page_theap(page); - #if MI_STAT>0 mi_theap_stat_counter_increase(theap, pages_retire, 1); - #endif page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); mi_assert_internal(pq >= theap->pages); const size_t index = pq - theap->pages; @@ -460,42 +450,13 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); return; // don't free after all - } + } } #endif _mi_page_free(page, pq); } -// free retired pages: we don't need to look at the entire queues -// since we only retire pages that are at the head position in a queue. -void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { - size_t min = MI_BIN_FULL; - size_t max = 0; - for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) { - mi_page_queue_t* pq = &theap->pages[bin]; - mi_page_t* page = pq->first; - if (page != NULL && page->retire_expire != 0) { - if (mi_page_all_free(page)) { - page->retire_expire--; - if (page->retire_expire == 0 || force) { - _mi_page_free(page, pq); - } - else { - // keep retired, update min/max - if (bin < min) min = bin; - if (bin > max) max = bin; - } - } - else { - page->retire_expire = 0; - } - } - } - theap->page_retired_min = min; - theap->page_retired_max = max; -} -/* static void mi_theap_collect_full_pages(mi_theap_t* theap) { // note: normally full pages get immediately abandoned and the full queue is always empty // this path is only used if abandoning is disabled due to a destroy-able theap or options @@ -516,7 +477,47 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { page = next; } } -*/ + +static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { + mi_assert_internal(page!=NULL && page->retire_expire!=0); + if (mi_page_all_free(page)) { + page->retire_expire--; + if (page->retire_expire == 0 || force) { + _mi_page_free(page, pq); + } + else { + // keep retired, update min/max + if (bin < *min) *min = bin; + if (bin > *max) *max = bin; + } + } + else { + page->retire_expire = 0; + } +} + +// free retired pages: we don't need to look at the entire queues +// since we only retire pages that are at the head position in a queue. +void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { + size_t min = MI_BIN_FULL; + size_t max = 0; + for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) { + mi_page_queue_t* pq = &theap->pages[bin]; + mi_page_t* page = pq->first; + for (int i = 0; iretire_expire!=0; i++) { + mi_page_t* next = page->next; + mi_page_try_retire(pq,page,bin,force,&min,&max); + page = next; + } + } + theap->page_retired_min = min; + theap->page_retired_max = max; + if (!theap->allow_page_abandon) { + mi_theap_collect_full_pages(theap); + } +} + + /* ----------------------------------------------------------- @@ -530,7 +531,7 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { #define MI_MIN_SLICES (2) static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) { - #if (MI_SECURE<3) + #if (MI_SECURE < 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif @@ -587,7 +588,7 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend) { - #if (MI_SECURE<3) + #if (MI_SECURE < 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif @@ -614,7 +615,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co Page initialize and extend the capacity ----------------------------------------------------------- */ -#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well. +#define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well. #if (MI_SECURE>=2) #define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many #else @@ -628,7 +629,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co // extra test in malloc? or cache effects?) static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); - #if (MI_SECURE<3) + #if (MI_SECURE < 2) mi_assert(page->free == NULL); mi_assert(page->local_free == NULL); if (page->free != NULL) return true; @@ -661,16 +662,30 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(extend < (1UL<<16)); // commit on demand? - if (page->slice_committed > 0) { + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { + // reduce extend if it commits more than an arena slice + if ((extend * bsize) > MI_ARENA_SLICE_SIZE) { + extend = _mi_divide_up(MI_ARENA_SLICE_SIZE, bsize); + } + // commit required size const size_t needed_size = (page->capacity + extend)*bsize; - const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE ); - if (needed_commit > page->slice_committed) { - mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0); - if (!_mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) { + mi_assert_internal(needed_size <= page_size); + size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size()); + #if MI_SECURE>=5 + // the previous alignup could extend the commit into the guard page; re-adjust if needed + const size_t page_size_commit = _mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() ); + if (needed_commit > page_size_commit) { + needed_commit = page_size_commit; + } + #endif + if (needed_commit > slice_committed) { + mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0); + if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) { return false; } - mi_assert_internal(needed_commit < UINT32_MAX); - page->slice_committed = (uint32_t)needed_commit; + mi_assert_internal(needed_commit <= UINT16_MAX * _mi_os_page_size()); + page->slice_pcommitted = (uint16_t)(needed_commit / _mi_os_page_size()); } } @@ -694,8 +709,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert(page != NULL); mi_assert(theap!=NULL); - page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap` - mi_page_set_theap(page, theap); + // page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap` + // mi_page_set_theap(page, theap); size_t page_size; uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start); @@ -704,8 +719,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->reserved > 0); #if (MI_PADDING || MI_ENCODE_FREELIST) page->keys[0] = _mi_theap_random_next(theap); + #if MI_PAGE_KEY_COUNT==2 page->keys[1] = _mi_theap_random_next(theap); #endif + #endif #if MI_DEBUG>2 if (page->memid.initially_zero) { mi_track_mem_defined(mi_page_start(page), mi_page_committed(page)); @@ -713,6 +730,8 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { } #endif + mi_assert_internal(page->heap != NULL); + mi_assert_internal(page->heap == _mi_theap_heap(theap)); mi_assert_internal(page->theap!=NULL); mi_assert_internal(page->theap == mi_page_theap(page)); mi_assert_internal(page->capacity == 0); @@ -726,8 +745,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(!mi_page_has_interior_pointers(page)); #if (MI_PADDING || MI_ENCODE_FREELIST) mi_assert_internal(page->keys[0] != 0); + #if MI_PAGE_KEY_COUNT==2 mi_assert_internal(page->keys[1] != 0); #endif + #endif mi_assert_expensive(mi_page_is_valid_init(page)); // initialize an initial free list @@ -833,7 +854,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, if (page == NULL) { _mi_theap_collect_retired(theap, false); // perhaps make a page available - page = mi_page_fresh(theap, pq); + page = mi_page_fresh(theap, pq); mi_assert_internal(page == NULL || mi_page_immediate_available(page)); if (page == NULL && first_try) { // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again @@ -854,63 +875,48 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, return page; } - - -// Find a page with free blocks of `size`. -static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) { - // mi_page_queue_t* pq = mi_page_queue(theap, size); +// Look for a page with free blocks of `size` but don't try to search or allocate +static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) { mi_assert_internal(!mi_page_queue_is_huge(pq)); - - // check the first page: we even do this with candidate search or otherwise we re-search every time mi_page_t* page = pq->first; - if mi_likely(page != NULL && mi_page_free_quick_collect(page)) { + if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) { + // fast path #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) { (void)mi_page_extend_free(theap, page); // ok if this fails mi_assert_internal(mi_page_immediate_available(page)); } + #else + MI_UNUSED(theap); #endif page->retire_expire = 0; - return page; // fast path + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; } else { - return mi_page_queue_find_free_ex(theap, pq, true); + return NULL; } } +// Find a page with free blocks of `size`. +static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) { + // mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(!mi_page_queue_is_huge(pq)); -/* ----------------------------------------------------------- - Users can register a deferred free function called - when the `free` list is empty. Since the `local_free` - is separate this is deterministically called after - a certain number of allocations. ------------------------------------------------------------ */ - -// The program should only install a single deferred free handler before doing allocation. -static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) -static _Atomic(void*) deferred_arg; - -void _mi_deferred_free(mi_theap_t* theap, bool force) { - theap->heartbeat++; - mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); - if (fun != NULL && !theap->tld->recurse) { - theap->tld->recurse = true; - void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); - fun(force, theap->heartbeat, arg); - theap->tld->recurse = false; - } -} - -void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { - mi_atomic_store_ptr_release(void,&deferred_arg, arg); - mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); + // check the first page: we even do this with candidate search or otherwise we re-search every time + mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq); + if (page==NULL) { + page = mi_page_queue_find_free_ex(theap, pq, true); + if (page==NULL) return NULL; + } + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; } - -/* ----------------------------------------------------------- - General allocation ------------------------------------------------------------ */ - // Huge pages contain just one block, and the segment contains just that page. // Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX) // so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. @@ -939,7 +945,6 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page return page; } - // Allocate a page // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { @@ -949,66 +954,103 @@ static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_align return NULL; } mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size)); + mi_page_t* page; // huge allocation? if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) { - return mi_huge_page_alloc(theap,size,huge_alignment,pq); + page = mi_huge_page_alloc(theap,size,huge_alignment,pq); } else { // otherwise find a page with free blocks in our size segregated queues #if MI_PADDING mi_assert_internal(size >= MI_PADDING_SIZE); #endif - return mi_find_free_page(theap, pq); + page = mi_page_queue_find_free(theap,pq); } + if (page==NULL) return NULL; + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; } -// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. -// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for -// very large requested alignments in which case we use a huge singleton page. -// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) -// to use 4 parameters which compiles better on msvc for the malloc fast path. -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept -{ - const bool zero = ((zero_huge_alignment & 1) != 0); - const size_t huge_alignment = (zero_huge_alignment & ~1); - #if !MI_THEAP_INITASNULL - mi_assert_internal(theap != NULL); - #endif +/* ----------------------------------------------------------- + Users can register a deferred free function called + when the `free` list is empty. Since the `local_free` + is separate this is deterministically called after + a certain number of allocations. +----------------------------------------------------------- */ - // initialize if necessary +// The program should only install a single deferred free handler before doing allocation. +static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) +static _Atomic(void*) deferred_arg; + +void _mi_deferred_free(mi_theap_t* theap, bool force) { + theap->heartbeat++; + mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); + if (fun != NULL && !theap->tld->recurse) { + theap->tld->recurse = true; + void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); + fun(force, theap->heartbeat, arg); + theap->tld->recurse = false; + } +} + +void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { + mi_atomic_store_ptr_release(void,&deferred_arg, arg); + mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); +} + + +/* ----------------------------------------------------------- + Admin +----------------------------------------------------------- */ + +static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) +{ if mi_unlikely(!mi_theap_is_initialized(theap)) { if (theap==&_mi_theap_empty_wrong) { // we were unable to allocate a theap for a first-class heap return NULL; } // otherwise we initialize the thread and its default theap - mi_thread_init(); - theap = _mi_theap_default(); - if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } - mi_assert_internal(_mi_theap_default()==theap); + theap = _mi_thread_init(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } } mi_assert_internal(mi_theap_is_initialized(theap)); // do administrative tasks every N generic mallocs - if mi_unlikely(++theap->generic_count >= 1000) { + if mi_unlikely(theap->generic_count >= 1000) { theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; - // call potential deferred free routines - _mi_deferred_free(theap, false); - // free retired pages - _mi_theap_collect_retired(theap, false); - - // collect every once in a while (10000 by default) + + // do a full theap collect every once in a while (10000 by default) const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); if (theap->generic_collect_count >= generic_collect) { theap->generic_collect_count = 0; mi_theap_collect(theap, false /* force? */); } + else { + // otherwise we do a mini-collect + _mi_deferred_free(theap, false); // call potential deferred free routines + _mi_theap_collect_retired(theap, false); // free retired pages + } } + return theap; +} + +/* ----------------------------------------------------------- + Generic allocation +----------------------------------------------------------- */ +static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) +{ + // initialize if necessary + theap = mi_malloc_generic_admin(theap); + if (theap==NULL) return NULL; + // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(theap, size, huge_alignment); if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more @@ -1028,7 +1070,7 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) - if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if (ppage!=NULL) { *ppage = page; } void* const p = _mi_page_malloc_zero(theap,page,size,zero); mi_assert_internal(p != NULL); @@ -1038,3 +1080,38 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm } return p; } + + +// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for +// very large requested alignments in which case we use a huge singleton page. +// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) +// to use 4 parameters which compiles better on msvc for the malloc fast path. +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept +{ + #if !MI_THEAP_INITASNULL + mi_assert_internal(theap != NULL); + #endif + const bool zero = ((zero_huge_alignment & 1) != 0); + const size_t huge_alignment = (zero_huge_alignment & ~1); + mi_page_t* page = NULL; + + // fast path objects that fit in a small page + if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); + return _mi_page_malloc_zero(theap,page,size,zero); + } + } + } + // otherwise fallback + return mi_malloc_generic_fallback(theap,size,zero, huge_alignment,ppage); +} diff --git a/system/lib/mimalloc/src/prim/emscripten/prim.c b/system/lib/mimalloc/src/prim/emscripten/prim.c index b52735ae270ff..992965d56f23b 100644 --- a/system/lib/mimalloc/src/prim/emscripten/prim.c +++ b/system/lib/mimalloc/src/prim/emscripten/prim.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen, Alon Zakai This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -61,7 +61,6 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) { extern void emmalloc_free(void*); int _mi_prim_free(void* addr, size_t size) { - if (size==0) return 0; emmalloc_free(addr); return 0; } @@ -77,7 +76,7 @@ extern void* emmalloc_memalign(size_t alignment, size_t size); int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; - // TODO: Track the highest address ever seen; first uses of it are zeroes. + // todo: Track the highest address ever seen; first uses of it are zeroes. // That assumes no one else uses sbrk but us (they could go up, // scribble, and then down), but we could assert on that perhaps. *is_zero = false; @@ -101,7 +100,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { MI_UNUSED(addr); MI_UNUSED(size); - // See TODO above. + // See todo above. *is_zero = false; return 0; } @@ -155,7 +154,8 @@ size_t _mi_prim_numa_node_count(void) { #include mi_msecs_t _mi_prim_clock_now(void) { - return emscripten_date_now(); + // todo: use a monotonic clock instead + return emscripten_date_now(); } @@ -212,7 +212,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default theap) -pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); +pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; static void mi_pthread_done(void* value) { if (value!=NULL) { @@ -221,18 +221,20 @@ static void mi_pthread_done(void* value) { } void _mi_prim_thread_init_auto_done(void) { - mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); + mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID); pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); } void _mi_prim_thread_done_auto_done(void) { - if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809 - pthread_key_delete(_mi_heap_default_key); + pthread_key_t key = _mi_heap_default_key; + if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809 + _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); } } void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { - if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD + if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD pthread_setspecific(_mi_heap_default_key, theap); } } diff --git a/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c b/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c index d66403a32ab00..48f97958218d9 100644 --- a/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c +++ b/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" - +#include "mimalloc/prim-tls.h" // _mi_thread_is_initialized #if defined(MI_MALLOC_OVERRIDE) #if !defined(__APPLE__) @@ -71,11 +71,12 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { } static void zone_free(malloc_zone_t* zone, void* p) { - if (mi_any_heap_contains(p)) { - mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free - } - else if (!is_mimalloc_zone(zone)) { // can happen due to interpose - zone->free(zone,p); + // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` + // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. + if mi_unlikely(!mi_cfree(p)) { + if (!is_mimalloc_zone(zone)) { // can happen due to interpose + zone->free(zone,p); + } } } @@ -225,6 +226,8 @@ static malloc_introspection_t mi_introspect = { #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__) .statistics = &intro_statistics, .zone_locked = &intro_zone_locked, +#endif +#if defined(MAC_OS_X_VERSION_10_12) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_12) && !defined(__ppc__) .reinit_lock = &intro_reinit_lock, #endif }; diff --git a/system/lib/mimalloc/src/prim/prim-tls.c b/system/lib/mimalloc/src/prim/prim-tls.c new file mode 100644 index 0000000000000..8c8b0c71fc17f --- /dev/null +++ b/system/lib/mimalloc/src/prim/prim-tls.c @@ -0,0 +1,252 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" + +// -------------------------------------------------------------------------- +// Implement fast access to the thread local storage for `_mi_theap_default()` +// and `mi_theap_cached()`. See `include/mimalloc/prim-tls.h` for more info +// on the TLS models. +// -------------------------------------------------------------------------- + +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + + +#if MI_TLS_MODEL_LOCAL +// the thread-local main theap for allocation +mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty; +// the last used non-main theap +mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty; +#endif + +mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL; + +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { + const mi_threadid_t tid = _mi_prim_thread_id(); + mi_assert_internal( (tid & MI_PAGE_FLAG_MASK) == 0 ); // mimalloc reserves the bottom 2 bits + return tid; +} + +// ---------------------------------------------------------------------------- +// Setting the default and cached theap +// ---------------------------------------------------------------------------- + +#if MI_TLS_MODEL_WIN32 + +// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) +// See for the offsets. +#if MI_SIZE_SIZE==4 +#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) +#else +#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) +#endif +#define MI_TLS_DIRECT_SLOTS (64) +#define MI_TLS_EXPANSION_SLOTS (1024) + +#if !MI_WIN_DIRECT_TLS +// We initially use the last of the expansion slots as the default NULL. +// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc +// before we are initialized :-( (but this seems quite unlikely). +// (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL) +#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT +#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) +#else +// With direct tls we need an initial NULL slot outside the expansion slots +#define MI_TLS_INITIAL_SLOT (5) // Arbitrary user pointer +#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) // unused +#endif + +// in case of errors assign fixed slots (but since we use EFAULT the program should fail anyways) +#define MI_TLS_ERROR_SLOT (5) // arbitrary user pointer +#define MI_TLS_ERROR_EXPANSION_SLOT (7) // environment pointer (only used for OS/2 emulation) + + +mi_decl_hidden mi_decl_cache_align _Atomic(size_t) _mi_theap_default_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT); + +static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES; +static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES; + +static bool mi_win_tls_slot_alloc(_Atomic(size_t)* slot, _Atomic(size_t)* extended, DWORD* raw_index) { + // always write slot before extended due to concurrent readers + const DWORD index = TlsAlloc(); + *raw_index = index; + if (index==TLS_OUT_OF_INDEXES) { + mi_atomic_store_release(slot,MI_TLS_ERROR_SLOT); + mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT); + return false; + } + else if (index= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT); + if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) { + mi_prim_tls_slot_set(slot, value); + } + else { + mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS); + TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed + } +} + +#elif MI_TLS_MODEL_PTHREADS + +// only for pthreads for now +mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); +mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); + +static void mi_theap_cached_key_destroy(void* theapv) { + mi_theap_t* theap = (mi_theap_t*)theapv; + if (theap!=NULL) { + _mi_theap_decref(theap); + } +} + +void _mi_tls_slots_init(void) { + mi_atomic_do_once { + pthread_key_t key; + if (_mi_pthread_key_create(&key,NULL,NULL)) { mi_atomic_store_release(&_mi_theap_default_key,key); } + if (_mi_pthread_key_create(&key,&mi_theap_cached_key_destroy,NULL)) { mi_atomic_store_release(&_mi_theap_cached_key,key); } + } +} + +void _mi_tls_slots_done(void) { + pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_theap_default_key,MI_PTHREAD_KEY_INVALID); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } + key = mi_atomic_exchange_relaxed(&_mi_theap_cached_key,MI_PTHREAD_KEY_INVALID); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } +} + +#elif MI_TLS_MODEL_FIXED + +void _mi_tls_slots_init(void) { + mi_atomic_do_once { + mi_theap_t* theap = _mi_theap_default(); + if (theap!=NULL) { + _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_DEFAULT, theap); + } + theap = _mi_theap_cached(); + if (theap!=NULL) { + _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_CACHED, theap); + } + } +} + +void _mi_tls_slots_done(void) { + // nothing +} + + +#else + +void _mi_tls_slots_init(void) { + // nothing +} + +void _mi_tls_slots_done(void) { + // nothing +} + +#endif + +void _mi_theap_cached_set(mi_theap_t* theap) { + mi_theap_t* prev = _mi_theap_cached(); + if (prev==theap) return; + // set + _mi_tls_slots_init(); + #if MI_TLS_MODEL_LOCAL + __mi_theap_cached = theap; + #elif MI_TLS_MODEL_FIXED + mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_CACHED, theap); + #elif MI_TLS_MODEL_WIN32 + mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_cached_slot), mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot), theap); + #elif MI_TLS_MODEL_PTHREADS + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); } + #endif + // update refcounts (so cached theap memory keeps available until no longer cached) + _mi_theap_incref(theap); + _mi_theap_decref(prev); +} + +void _mi_theap_default_set(mi_theap_t* theap) { + mi_assert_internal(theap != NULL); + mi_assert_internal(theap->tld != NULL); + mi_assert_internal(mi_theap_matches_thread(theap)); + _mi_tls_slots_init(); + #if MI_TLS_MODEL_LOCAL + __mi_theap_default = theap; + #elif MI_TLS_MODEL_FIXED + mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_DEFAULT, theap); + #elif MI_TLS_MODEL_WIN32 + mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_default_slot), mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot), theap); + #elif MI_TLS_MODEL_PTHREADS + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); } + #endif + + // set theap main if needed + if (mi_theap_is_initialized(theap)) { + // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again) + _mi_prim_thread_associate_default_theap(theap); + } +} diff --git a/system/lib/mimalloc/src/prim/prim.c b/system/lib/mimalloc/src/prim/prim.c index 9e2afdfc880a8..7f31c994dae78 100644 --- a/system/lib/mimalloc/src/prim/prim.c +++ b/system/lib/mimalloc/src/prim/prim.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/system/lib/mimalloc/src/prim/readme.md b/system/lib/mimalloc/src/prim/readme.md index 380dd3a717841..02cb1b90c8c0e 100644 --- a/system/lib/mimalloc/src/prim/readme.md +++ b/system/lib/mimalloc/src/prim/readme.md @@ -6,4 +6,7 @@ This is the portability layer where all primitives needed from the OS are define - `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform (and on macOS, `osx/prim.c` defers to `unix/prim.c`). +- `include/mimalloc/prim-tls.h`: primitive TLS model (thread local, pthreads, etc) +- `prim-tls.c`: TLS model implementation + Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file diff --git a/system/lib/mimalloc/src/prim/unix/prim.c b/system/lib/mimalloc/src/prim/unix/prim.c index 3b817c8363f41..1838901d56fa6 100644 --- a/system/lib/mimalloc/src/prim/unix/prim.c +++ b/system/lib/mimalloc/src/prim/unix/prim.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -79,7 +79,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MADV_FREE POSIX_MADV_FREE #endif -#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this? +#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // todo: can we query the OS for this? //------------------------------------------------------------------------------------ // Use syscalls for some primitives to allow for libraries that override open/read/close etc. @@ -235,7 +235,7 @@ static size_t unix_detect_virtual_address_bits(void) { if (fd >= 0) { char buf[2048]; const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); - mi_prim_close(fd); + mi_prim_close(fd); if ((nread >= 1) && (nread <= (ssize_t)sizeof(buf))) { if (_mi_strnstr(buf, nread, "sv39")) { return 39; } else if (_mi_strnstr(buf, nread, "sv48")) { return 48; } @@ -294,7 +294,7 @@ int _mi_prim_free(void* addr, size_t size ) { // return errno on failure static int unix_madvise(void* addr, size_t size, int advice) { - #if defined(__sun) + #if defined(__sun) || defined(_AIX) const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) return (res==0 ? 0 : errno); #elif defined(__QNX__) @@ -376,6 +376,10 @@ static int unix_mmap_fd(void) { #endif } +#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) +#define MI_OS_HAS_HUGE_PAGES 1 +#endif + static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON @@ -393,6 +397,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif // huge page allocation + #if MI_OS_HAS_HUGE_PAGES if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) { static _Atomic(size_t) large_page_try_ok; // = 0; size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); @@ -447,7 +452,10 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec } } } - } + } // huge pages + #else + MI_UNUSED(large_only); + #endif // regular allocation if (p == NULL) { *is_large = false; @@ -455,15 +463,17 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #if !defined(MI_NO_THP) if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { - // *is_large = true; // possibly - }; + if (_mi_os_canuse_thp()) { + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + // *is_large = true; // possibly + }; + } #elif defined(__sun) struct memcntl_mha cmd = {0}; cmd.mha_pagesize = _mi_os_large_page_size(); @@ -495,7 +505,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm //--------------------------------------------- static void unix_mprotect_hint(int err) { - #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page + #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc page if (err == ENOMEM) { _mi_warning_message("The next warning may be caused by a low memory map limit.\n" " On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n" @@ -623,9 +633,9 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo bool is_large = true; *is_zero = true; *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large); - if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes + if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_INTPTR_SIZE - 1)) { // at most 63 nodes unsigned long numa_mask = (1UL << numa_node); - // TODO: does `mbind` work correctly for huge OS pages? should we + // todo: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); @@ -656,9 +666,9 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo size_t _mi_prim_numa_node(void) { #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu) - unsigned long node = 0; - unsigned long ncpu = 0; - long err = syscall(SYS_getcpu, &ncpu, &node, NULL); + unsigned int node = 0; + unsigned int ncpu = 0; + int err = syscall(SYS_getcpu, &ncpu, &node, NULL); if (err != 0) return 0; return node; #else @@ -668,13 +678,16 @@ size_t _mi_prim_numa_node(void) { size_t _mi_prim_numa_node_count(void) { char buf[128]; - unsigned node = 0; - for(node = 0; node < 256; node++) { + unsigned last_found = 0; + for(unsigned node = 1; node < 256; node++) { // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) - _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); - if (mi_prim_access(buf,R_OK) != 0) break; + _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node); + if (mi_prim_access(buf,R_OK) != 0) { + if (node - last_found > 4) break; // allow some sparseness of nodes but not more than 4 + } + else { last_found = node; } // highest found node } - return (node+1); + return last_found + 1; } #elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 @@ -700,7 +713,7 @@ size_t _mi_prim_numa_node_count(void) { #elif defined(__DragonFly__) size_t _mi_prim_numa_node(void) { - // TODO: DragonFly does not seem to provide any userland means to get this information. + // todo: DragonFly does not seem to provide any userland means to get this information. return 0ul; } @@ -733,7 +746,7 @@ size_t _mi_prim_numa_node_count(void) { // low resolution timer static mi_msecs_t mi_prim_clock_now_lowres(void) { const int64_t ticks = (int64_t)clock(); - #if !defined(CLOCKS_PER_SEC) + #if !defined(CLOCKS_PER_SEC) return ticks; #else if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) { @@ -755,12 +768,12 @@ mi_msecs_t _mi_prim_clock_now(void) { #else const clockid_t clockid = CLOCK_REALTIME; #endif - struct timespec t; + struct timespec t; if (clock_gettime(clockid,&t) == 0) { return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L); } - #endif - return mi_prim_clock_now_lowres(); + #endif + return mi_prim_clock_now_lowres(); } @@ -788,41 +801,46 @@ static mi_msecs_t timeval_secs(const struct timeval* tv) { void _mi_prim_process_info(mi_process_info_t* pinfo) { struct rusage rusage; - getrusage(RUSAGE_SELF, &rusage); - pinfo->utime = timeval_secs(&rusage.ru_utime); - pinfo->stime = timeval_secs(&rusage.ru_stime); -#if !defined(__HAIKU__) - pinfo->page_faults = rusage.ru_majflt; -#endif -#if defined(__HAIKU__) - // Haiku does not have (yet?) a way to - // get these stats per process - thread_info tid; - area_info mem; - ssize_t c; - get_thread_info(find_thread(0), &tid); - while (get_next_area_info(tid.team, &c, &mem) == B_OK) { - pinfo->peak_rss += mem.ram_size; - } - pinfo->page_faults = 0; -#elif defined(__APPLE__) - pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes - #ifdef MACH_TASK_BASIC_INFO - struct mach_task_basic_info info; - mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - pinfo->current_rss = (size_t)info.resident_size; - } - #else - struct task_basic_info info; - mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - pinfo->current_rss = (size_t)info.resident_size; + if (getrusage(RUSAGE_SELF, &rusage) == 0) { + pinfo->utime = timeval_secs(&rusage.ru_utime); + pinfo->stime = timeval_secs(&rusage.ru_stime); + #if !defined(__HAIKU__) + pinfo->page_faults = rusage.ru_majflt; + #endif + #if defined(__APPLE__) + pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes + #else + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB + #endif } + + #if defined(__HAIKU__) + // Haiku does not have (yet?) a way to + // get these stats per process + thread_info tid; + if (get_thread_info(find_thread(0), &tid) == B_OK) { + area_info mem; + ssize_t c; + while (get_next_area_info(tid.team, &c, &mem) == B_OK) { + pinfo->peak_rss += mem.ram_size; + } + } + pinfo->page_faults = 0; + #elif defined(__APPLE__) + #ifdef MACH_TASK_BASIC_INFO + struct mach_task_basic_info info; + mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #else + struct task_basic_info info; + mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #endif #endif -#else - pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB -#endif // use defaults for commit } @@ -880,8 +898,8 @@ int _mi_prim_getenv(const char* name, char* result, size_t result_size) { const char* s = env[i]; if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive // found it - _mi_strlcpy(result, s + len + 1, result_size); - return 1; // success + if (!_mi_strlcpy(result, s + len + 1, result_size)) return -1; + return 1; // success } } return 0; // not found @@ -902,8 +920,8 @@ int _mi_prim_getenv(const char* name, char* result, size_t result_size) { buf[len] = 0; s = getenv(buf); } - if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found - _mi_strlcpy(result, s, result_size); + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found + if (!_mi_strlcpy(result, s, result_size)) return -1; return 1; // success } #endif // !MI_USE_ENVIRON @@ -933,7 +951,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { return true; } -#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829) +#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__) // also for old apple versions < 10.7 (issue #829) #include #include @@ -965,7 +983,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { size_t count = 0; while(count < buf_len) { ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count); - if (ret<=0) { + if (ret==0) { + break; + } + else if (ret<0) { if (errno!=EAGAIN && errno!=EINTR) break; } else { @@ -993,7 +1014,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default theap) -pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); +pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; static void mi_pthread_done(void* value) { if (value!=NULL) { @@ -1002,26 +1023,25 @@ static void mi_pthread_done(void* value) { } void _mi_prim_thread_init_auto_done(void) { - mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); - const int err = pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); - if (err!=0) { - _mi_error_message(err,"unable to create a pthread thread local key (error %d (0x%x))", err, err); - _mi_heap_default_key = (pthread_key_t)(-1); - }; + mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID); + pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); } void _mi_prim_thread_done_auto_done(void) { - if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809 - pthread_key_delete(_mi_heap_default_key); + pthread_key_t key = _mi_heap_default_key; + if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809 + _mi_heap_default_key = MI_PTHREAD_KEY_INVALID; + pthread_key_delete(key); } } void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) { - if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD + if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD pthread_setspecific(_mi_heap_default_key, theap); } } + #else void _mi_prim_thread_init_auto_done(void) { diff --git a/system/lib/mimalloc/src/prim/windows/prim.c b/system/lib/mimalloc/src/prim/windows/prim.c index 6c4a51dd9e55c..0dfc422026bc1 100644 --- a/system/lib/mimalloc/src/prim/windows/prim.c +++ b/system/lib/mimalloc/src/prim/windows/prim.c @@ -13,9 +13,9 @@ terms of the MIT license. A copy of the license can be found in the file #include // fputs, stderr #include // atexit -// xbox has no console IO -#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM) -#define MI_HAS_CONSOLE_IO +// xbox has no console IO and cannot use LoadLibrary or GetModuleHandle +#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) +#define MI_WIN_DESKTOP 1 #endif //--------------------------------------------- @@ -87,6 +87,34 @@ typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalM typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation); +// Load a library +static HMODULE mi_win_loadlibrary(const TCHAR* library) { + #if MI_WIN_DESKTOP + return LoadLibrary(library); + #else + return LoadPackagedLibrary(library, 0); + #endif +} + +// Get a library handle (and possibly load it) +static HMODULE mi_win_getlibrary(const TCHAR* library, bool* should_free) { + #if MI_WIN_DESKTOP + // avoid calling LoadLibrary for "kernel32", "ntdll", and "kernelbase" (also to avoid hitting the loader lock) + HMODULE mod = GetModuleHandle(library); + if (mod!=NULL) { + *should_free = false; + return mod; + } + #endif + *should_free = true; + return mi_win_loadlibrary(library); +} + +static void mi_win_freelibrary(HMODULE mod, bool should_free) { + if (should_free) { + FreeLibrary(mod); + } +} //--------------------------------------------- // Enable large page support dynamically (if possible) @@ -103,15 +131,17 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size) unsigned long err = 0; HANDLE token = NULL; BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); + err = GetLastError(); if (ok) { TOKEN_PRIVILEGES tp; ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); + err = GetLastError(); if (ok) { tp.PrivilegeCount = 1; tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); + err = GetLastError(); if (ok) { - err = GetLastError(); ok = (err == ERROR_SUCCESS); if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) { *large_page_size = (*pGetLargePageMinimum)(); @@ -121,17 +151,19 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size) CloseHandle(token); } if (!ok) { - if (err == 0) err = GetLastError(); + if (err == 0) { err = GetLastError(); } _mi_warning_message("cannot enable large OS page support, error %lu\n", err); } return (ok!=0); } static bool win_enable_large_os_pages(size_t* large_page_size) { + static size_t win_large_page_size = 0; mi_atomic_do_once { - win_enable_large_os_pages_once(large_page_size); + win_enable_large_os_pages_once(&win_large_page_size); } - return (_mi_os_large_page_size() > 0); + if (large_page_size != NULL) { *large_page_size = win_large_page_size; } + return (win_large_page_size > 0); } @@ -162,21 +194,22 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) } // get the VirtualAlloc2 function - HINSTANCE hDll = LoadLibrary(TEXT("kernelbase.dll")); + bool hDllFree; + HINSTANCE hDll = mi_win_getlibrary(TEXT("kernelbase.dll"), &hDllFree); if (hDll != NULL) { // use VirtualAlloc2FromApp if possible as it is available to Windows store apps pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); - FreeLibrary(hDll); + mi_win_freelibrary(hDll, hDllFree); } // NtAllocateVirtualMemoryEx is used for huge page allocation - hDll = LoadLibrary(TEXT("ntdll.dll")); + hDll = mi_win_getlibrary(TEXT("ntdll.dll"), &hDllFree); if (hDll != NULL) { pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); - FreeLibrary(hDll); + mi_win_freelibrary(hDll, hDllFree); } // Try to use Win7+ numa API - hDll = LoadLibrary(TEXT("kernel32.dll")); + hDll = mi_win_getlibrary(TEXT("kernel32.dll"), &hDllFree); if (hDll != NULL) { pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); @@ -205,7 +238,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) win_minor_version = version.dwMinorVersion; } } - FreeLibrary(hDll); + mi_win_freelibrary(hDll, hDllFree); } // Enable large/huge OS page support? if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { @@ -228,8 +261,9 @@ int _mi_prim_free(void* addr, size_t size ) { // the memory region returned by VirtualAlloc; in that case we need to free using // the start of the region. MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info); - VirtualQuery(addr, &info, sizeof(info)); - if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) { + err = (VirtualQuery(addr, &info, sizeof(info)) == 0); + if (err) { errcode = GetLastError(); } + if (!err && info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) { errcode = 0; err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); if (err) { errcode = GetLastError(); } @@ -525,8 +559,9 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { static LARGE_INTEGER mfreq; // = 0 if (mfreq.QuadPart == 0LL) { LARGE_INTEGER f; - QueryPerformanceFrequency(&f); - mfreq.QuadPart = f.QuadPart/1000LL; + if (QueryPerformanceFrequency(&f)) { + mfreq.QuadPart = f.QuadPart/1000LL; + } if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; } return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); @@ -534,8 +569,12 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { mi_msecs_t _mi_prim_clock_now(void) { LARGE_INTEGER t; - QueryPerformanceCounter(&t); - return mi_to_msecs(t); + if (QueryPerformanceCounter(&t)) { + return mi_to_msecs(t); + } + else { + return 0; + } } @@ -562,29 +601,31 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) FILETIME ut; FILETIME st; FILETIME et; - GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); - pinfo->utime = filetime_msecs(&ut); - pinfo->stime = filetime_msecs(&st); + if (GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut)) { + pinfo->utime = filetime_msecs(&ut); + pinfo->stime = filetime_msecs(&st); + } // load psapi on demand - mi_atomic_do_once { - HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); + mi_atomic_do_once{ + HINSTANCE hDll = mi_win_loadlibrary(TEXT("psapi.dll")); if (hDll != NULL) { pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); - // FreeLibrary(hDll); // don't free + // mi_win_freelibrary(hDll, true); // don't free } } // get process info - PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); if (pGetProcessMemoryInfo != NULL) { - pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); + PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); + if (pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) { + pinfo->current_rss = (size_t)info.WorkingSetSize; + pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; + pinfo->current_commit = (size_t)info.PagefileUsage; + pinfo->peak_commit = (size_t)info.PeakPagefileUsage; + pinfo->page_faults = (size_t)info.PageFaultCount; + } } - pinfo->current_rss = (size_t)info.WorkingSetSize; - pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; - pinfo->current_commit = (size_t)info.PagefileUsage; - pinfo->peak_commit = (size_t)info.PeakPagefileUsage; - pinfo->page_faults = (size_t)info.PageFaultCount; } //---------------------------------------------------------------- @@ -596,32 +637,37 @@ void _mi_prim_out_stderr( const char* msg ) // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. if (!_mi_preloading()) { - // _cputs(msg); // _cputs cannot be used as it aborts when failing to lock the console static HANDLE hcon = INVALID_HANDLE_VALUE; + #if MI_WIN_DESKTOP static bool hconIsConsole = false; + #endif if (hcon == INVALID_HANDLE_VALUE) { - hcon = GetStdHandle(STD_ERROR_HANDLE); - #ifdef MI_HAS_CONSOLE_IO + hcon = GetStdHandle(STD_ERROR_HANDLE); // returns NULL if no stderr is available + #if MI_WIN_DESKTOP + if (hcon==NULL) { + AttachConsole(ATTACH_PARENT_PROCESS); // if started from a parent console, try to attach to that + hcon = GetStdHandle(STD_ERROR_HANDLE); + } CONSOLE_SCREEN_BUFFER_INFO sbi; - hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); + hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); #endif } const size_t len = _mi_strlen(msg); if (len > 0 && len < UINT32_MAX) { DWORD written = 0; - if (hconIsConsole) { - #ifdef MI_HAS_CONSOLE_IO - WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + if (hcon != NULL && hcon != INVALID_HANDLE_VALUE) { + #if MI_WIN_DESKTOP + if (hconIsConsole) { + WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + } + else #endif + { + // use direct write in case stderr was redirected + WriteFile(hcon, msg, (DWORD)len, &written, NULL); + } } - else if (hcon != INVALID_HANDLE_VALUE) { - // use direct write if stderr was redirected - WriteFile(hcon, msg, (DWORD)len, &written, NULL); - } - else { - // finally fall back to fputs after all - fputs(msg, stderr); - } + // don't fall back to fputs or _cputs as the crt can have it locked } } } @@ -673,10 +719,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { mi_assert(buf_len <= ULONG_MAX); if (buf_len > ULONG_MAX) return false; mi_atomic_do_once { - HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll")); + HINSTANCE hDll = mi_win_loadlibrary(TEXT("bcrypt.dll")); if (hDll != NULL) { pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); - // FreeLibrary(hDll); // don't free + // mi_win_freelibrary(hDll); // don't free } } if (pBCryptGenRandom == NULL) return false; @@ -737,15 +783,18 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`). ------------------------------------------------------------------------- */ #if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS) - #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER) - #define MI_WIN_INIT_USE_CRT_TLS 1 + #if defined(__INTEL_LLVM_COMPILER) || defined(__INTEL_COMPILER) + #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* needed for Intel ICX, see issue #1268 */ #else - #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */ + #define MI_WIN_INIT_USE_CRT_TLS 1 /* default */ #endif #endif #if defined(MI_WIN_INIT_USE_CRT_TLS) - #define MI_PRIM_HAS_PROCESS_ATTACH 1 + #if !defined(__MINGW32__) || defined(MI_MINGW_UCRT64) // on mingw without UCRT use the constructor attribute (in `src/prim/prim.c`) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + #endif + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. void _mi_prim_thread_init_auto_done(void) {} void _mi_prim_thread_done_auto_done(void) {} @@ -818,10 +867,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #endif typedef int (mi_cdecl* mi_crt_callback_t)(void); - #if defined(_WIN64) + + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init") #pragma const_seg(".CRT$XLB") extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; @@ -834,10 +885,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const mi_crt_callback_t _mi_crt_callback_init[]; const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; #pragma const_seg() - #else + #elif defined(_MSC_VER) // 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init") #pragma data_seg(".CRT$XLB") PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; #pragma data_seg() @@ -847,6 +899,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #pragma data_seg(".CRT$XIB") mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; + __attribute__((used, section(".CRT$XIB"))) mi_crt_callback_t _mi_crt_callback_init = &mi_crt_init; #endif #if defined(__cplusplus) @@ -916,7 +974,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern "C" { #endif - #if defined(_WIN64) + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -928,7 +986,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; #pragma const_seg() - #else + #elif defined(_MSC_VER) // 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") @@ -938,6 +996,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #pragma data_seg(".CRT$XLY") PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; #endif #if defined(__cplusplus) @@ -985,7 +1048,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern "C" { #endif - #if defined(_WIN64) + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -997,16 +1060,21 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; #pragma const_seg() - #else + #elif defined(_MSC_VER) // 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") #pragma data_seg(".CRT$XLB") PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; #pragma data_seg() - #pragma data_seg(".CRT$XIY") + #pragma data_seg(".CRT$XLY") PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; #endif #if defined(__cplusplus) @@ -1027,21 +1095,31 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { // See #define MI_PRIM_HAS_PROCESS_ATTACH 1 - static int mi_process_attach(void) { + static int mi_cdecl mi_crt_init(void) { mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); atexit(&_mi_auto_process_done); return 0; } - typedef int(*mi_crt_callback_t)(void); - #if defined(_WIN64) - #pragma comment(linker, "/INCLUDE:_mi_tls_callback") - #pragma section(".CRT$XIU", long, read) - #else - #pragma comment(linker, "/INCLUDE:__mi_tls_callback") + + #if defined(__cplusplus) + extern "C" { + #endif + typedef int (mi_cdecl* mi_crt_callback_t)(void); + #if defined(_WIN64) // 64-bit + #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init") + #pragma const_seg(".CRT$XIU") + extern const mi_crt_callback_t _mi_crt_callback_init[]; + const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma const_seg() + #else // 32-bit + #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init") + #pragma data_seg(".CRT$XIU") + mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma data_seg() + #endif + #if defined(__cplusplus) + } #endif - #pragma data_seg(".CRT$XIU") - mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach }; - #pragma data_seg() #endif // use the fiber api for calling `_mi_thread_done`. diff --git a/system/lib/mimalloc/src/random.c b/system/lib/mimalloc/src/random.c index 75b5cfbb0bd44..464627acfb177 100644 --- a/system/lib/mimalloc/src/random.c +++ b/system/lib/mimalloc/src/random.c @@ -134,8 +134,9 @@ static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { mi_assert_internal(mi_random_is_initialized(ctx)); - mi_assert_internal(ctx != ctx_new); - chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new); + mi_assert_internal(ctx != ctx_new); + const uintptr_t nonce_rnd = _mi_random_next(ctx); + chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); } uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { @@ -193,6 +194,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { ctx->weak = false; } chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); + _mi_memzero(key, sizeof(key)); } void _mi_random_init(mi_random_ctx_t* ctx) { diff --git a/system/lib/mimalloc/src/static.c b/system/lib/mimalloc/src/static.c index 2383f65961357..3554a4bc679d6 100644 --- a/system/lib/mimalloc/src/static.c +++ b/system/lib/mimalloc/src/static.c @@ -24,7 +24,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "alloc-aligned.c" #include "alloc-posix.c" #include "arena.c" -#include "arena-meta.c" #include "bitmap.c" #include "heap.c" #include "init.c" @@ -35,9 +34,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "page-map.c" #include "random.c" #include "stats.c" +#include "subproc.c" #include "theap.c" #include "threadlocal.c" #include "prim/prim.c" +#include "prim/prim-tls.c" #if MI_OSX_ZONE #include "prim/osx/alloc-override-zone.c" #endif diff --git a/system/lib/mimalloc/src/stats.c b/system/lib/mimalloc/src/stats.c index 5cc3bbfe14dbc..9220bfec2e88f 100644 --- a/system/lib/mimalloc/src/stats.c +++ b/system/lib/mimalloc/src/stats.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -8,7 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-stats.h" #include "mimalloc/internal.h" #include "mimalloc/atomic.h" -#include "mimalloc/prim.h" +#include "mimalloc/prim.h" // _mi_prim_clock_now, mi_process_info_t +#include "mimalloc/prim-tls.h" #include // memset @@ -68,14 +69,17 @@ void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // adjust atomically + const size_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); mi_atomic_addi64_relaxed(&stat->current, amount); - mi_atomic_addi64_relaxed(&stat->total, amount); + const size_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); + if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } } static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; stat->current += amount; - stat->total += amount; + if (stat->total==stat->peak) { stat->peak += amount; } + stat->total += amount; } void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) { @@ -96,15 +100,17 @@ void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) { if (stat==src) return; mi_atomic_void_addi64_relaxed(&stat->total, &src->total); - const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current); + const int64_t src_peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->peak); + const int64_t src_current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->current); + const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current); // Global current plus thread peak approximates new global peak - // note: peak scores do really not work across threads. + // note: peak scores do not really work across threads. // we used to just add them together but that often overestimates in practice. // similarly, max does not seem to work well. The current approach // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 - // for a longer description. - mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak); + // for a longer description. + mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src_peak); } static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) { @@ -145,7 +151,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // unit > 0 : size in binary bytes // unit == 0: count as decimal // unit < 0 : count in binary -static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { +static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, bool limitwidth) { char buf[32]; _mi_memzero_var(buf); int len = 32; const char* suffix = (unit <= 0 ? " " : "B"); @@ -170,12 +176,17 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } - _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf); + if (limitwidth) { + _mi_fprintf(out, arg, "%12s", buf); + } + else { + _mi_fprintf(out, arg, "%s", buf); + } } static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { - mi_printf_amount(n,unit,out,arg,NULL); + mi_printf_amount(n,unit,out,arg,true); } static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { @@ -209,7 +220,7 @@ static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64 } if (stat->current != 0) { _mi_fprintf(out, arg, " "); - _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok)); + _mi_fprintf(out, arg, "%s", (notok == NULL ? "not all freed" : notok)); _mi_fprintf(out, arg, "\n"); } else { @@ -334,10 +345,10 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_ _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); - mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s"); + mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); if (peak_commit > 0) { _mi_fprintf(out, arg, ", peak commit: "); - mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); + mi_printf_amount((int64_t)peak_commit, 1, out, arg, false); } _mi_fprintf(out, arg, "\n"); } @@ -408,12 +419,13 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, ""); mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg); _mi_fprintf(out, arg, "\n"); - - mi_print_header("process", out, arg); - mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); - _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); - mi_process_info_print_out(out, arg); } + + mi_print_header("process", out, arg); + mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); + _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + mi_process_info_print_out(out, arg); + _mi_fprintf(out, arg, "\n"); } @@ -435,7 +447,7 @@ void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) { mi_assert_internal(to != NULL && from != NULL); if (to == from) return; mi_stats_add(to, from); - _mi_memzero(from, sizeof(mi_stats_t)); + mi_stats_init(from); // zero field and keep the header } static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept { @@ -452,11 +464,16 @@ static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) { else return mi_stats_merge_theap_to_heap(theap); } +static const mi_stats_t* mi_theap_get_stats(mi_theap_t* theap) { + return &theap->stats; +} + // deprecated void mi_stats_reset(void) mi_attr_noexcept { if (!mi_theap_is_initialized(_mi_theap_default())) return; - mi_heap_get_stats(mi_heap_main()); - mi_heap_stats_merge_to_subproc(mi_heap_main()); + mi_heap_t* heap_main = mi_heap_main(); + mi_heap_get_stats(heap_main); + mi_heap_stats_merge_to_subproc(heap_main); } @@ -483,6 +500,10 @@ void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* if (subproc==NULL) return; mi_heap_print_visit_info_t vinfo = { out, arg }; mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo); + if (subproc->theap_meta!=NULL) { + _mi_stats_print("meta", subproc->subproc_seq, &subproc->theap_meta->stats, out, arg); + } + // if (subproc->theap_meta!=NULL) { _mi_stats_merge_into(&subproc->stats, &subproc->theap_meta->stats); } _mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg); } @@ -551,8 +572,10 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s pinfo.elapsed = _mi_clock_end(mi_process_start); { const mi_subproc_t* subproc = _mi_subproc_main(); if (subproc!=NULL) { - pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current))); - pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak))); + const int64_t current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)); + const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)); + pinfo.current_commit = (current < 0 ? 0 : (current < PTRDIFF_MAX ? (size_t)current : PTRDIFF_MAX)); + pinfo.peak_commit = (peak < 0 ? 0 : (peak < PTRDIFF_MAX ? (size_t)peak : PTRDIFF_MAX)); } } pinfo.current_rss = pinfo.current_commit; @@ -604,6 +627,9 @@ bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept { return mi_stats_copy(stats, mi_heap_get_stats(heap)); } +bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept { + return mi_stats_copy(stats, mi_theap_get_stats(theap)); +} static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) { mi_stats_t* stats = (mi_stats_t*)arg; @@ -617,6 +643,8 @@ bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr if (subproc == NULL) return false; if (!mi_stats_copy(stats, &subproc->stats)) return false; mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats); + // hide meta data stats + // if (subproc->theap_meta!=NULL) { mi_stats_add_into(stats, &subproc->theap_meta->stats); } return true; } @@ -725,9 +753,6 @@ static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* nam mi_json_buf_print_value(hbuf, name, stat->total); } -#define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat); -#define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat); - static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept { if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL; mi_json_buf_t hbuf = { NULL, 0, 0, true }; @@ -766,7 +791,13 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, mi_json_buf_print(&hbuf, " },\n"); // statistics + #define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat); + #define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat); + MI_STAT_FIELDS() + + #undef MI_STAT_COUNT + #undef MI_STAT_COUNTER // size bins mi_json_buf_print(&hbuf, " \"malloc_bins\": [\n"); @@ -785,7 +816,7 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, } mi_json_buf_print(&hbuf, " ]\n"); mi_json_buf_print(&hbuf, "}\n"); - if (hbuf.used >= hbuf.size) { + if (hbuf.used + 1 >= hbuf.size) { // failed if (hbuf.can_realloc) { mi_free(hbuf.buf); } return NULL; diff --git a/system/lib/mimalloc/src/subproc.c b/system/lib/mimalloc/src/subproc.c new file mode 100644 index 0000000000000..7cce9edd75396 --- /dev/null +++ b/system/lib/mimalloc/src/subproc.c @@ -0,0 +1,327 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +// pre-allocate the main subprocess structure. +static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero; +static mi_subproc_t* mi_subprocs = NULL; +static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER; + + +/* ----------------------------------------------------------- + Meta-data allocation + We allocate thread local data and theaps through a dedicated + theap `subproc.theap_meta` which uses a detached tld with + a detached thread id. The initial theap_meta is statically + allocated and can thus be used to allocate on an as yet + uninitialized thread or process. + We need to take a lock though to allocate safely on the + detached `theap_meta`. +----------------------------------------------------------- */ + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc(subproc->theap_meta, size); + if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); } + } + return p; +} + +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t aligned, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc_aligned(subproc->theap_meta, size, aligned); + if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); } + } + return p; +} + +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + // note: since we take a meta lock we cannot use `mi_theap_rezalloc` as that could call `mi_free` which + // can call `mi_stat_free` which would try to take the meta lock again. See issue #1358. + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc(subproc->theap_meta, newsize); + } + if (p!=NULL) { + if (oldp!=NULL) { + const size_t oldsize = mi_usable_size(oldp); + const size_t copysize = (newsize < oldsize ? newsize : oldsize); + _mi_memcpy(p,oldp,copysize); + if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); } + else { mi_free(oldp); } + } + if (memid!=NULL) { *memid = _mi_memid_create_malloc(p,newsize,true); } + } + else { + if (memid!=NULL) { *memid = _mi_memid_none(); } + } + return p; +} + +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) { + if (p==NULL || mi_memid_needs_no_free(memid)) return; + if (memid.memkind == MI_MEM_MALLOC) { + mi_free(p); + } + else { + mi_assert_internal(subproc!=NULL); + _mi_arenas_free(subproc, p, _mi_memid_size(memid), memid); + } +} + +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) { + if (page==NULL) return false; + mi_theap_t* theap = page->theap; + return (theap != NULL && theap == subproc->theap_meta); +} + + +/* ----------------------------------------------------------- + Sub process helpers +----------------------------------------------------------- */ + +mi_subproc_t* _mi_subproc_main(void) { + return &mi_process_subproc_main; +} + +bool _mi_subproc_is_main(mi_subproc_t* subproc) { + return (subproc == &mi_process_subproc_main); +} + +mi_subproc_t* _mi_subproc(void) { + mi_theap_t* theap = _mi_theap_default(); + if (theap == NULL || theap->tld == NULL) { // see issue #1289 + return _mi_subproc_main(); + } + else { + return theap->tld->subproc; + } +} + +mi_heap_t* mi_heap_main(void) { + return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init +} + + +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { + return (mi_subproc_t*)(subproc_id._mi_subproc_id); +} + +mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) { + mi_subproc_id_t id = { subproc }; + return id; +} + +mi_subproc_id_t mi_subproc_main(void) { + return _mi_subproc_to_id(_mi_subproc_main()); +} + +mi_subproc_id_t mi_subproc_current(void) { + return _mi_subproc_to_id(_mi_subproc()); +} + + +/* ----------------------------------------------------------- + Sub process creation +----------------------------------------------------------- */ + + +static mi_subproc_t* mi_subproc_init(mi_subproc_t* subproc, mi_subproc_t* parent) { + static _Atomic(size_t) subproc_total_count; + subproc->parent = parent; + subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count); + mi_stats_header_init(&subproc->stats); + mi_lock_init(&subproc->arena_reserve_lock); + mi_lock_init(&subproc->heaps_lock); + mi_lock_init(&subproc->theap_meta_lock); + mi_lock(&mi_subprocs_lock) { + // push on subproc list + subproc->next = mi_subprocs; + if (mi_subprocs!=NULL) { mi_subprocs->prev = subproc; } + mi_subprocs = subproc; + } + return subproc; +} + +mi_subproc_id_t mi_subproc_new(void) { + mi_thread_init(); + mi_subproc_t* const parent = _mi_subproc(); + mi_memid_t memid; + mi_subproc_t* const subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t), &memid); + if (subproc == NULL) { return _mi_subproc_to_id(NULL); } + subproc->memid = memid; + + mi_memid_t theap_memid; + mi_theap_t* const theap_meta = (mi_theap_t*)_mi_meta_zalloc(parent, sizeof(mi_theap_t), &theap_memid); + if (theap_meta==NULL) { + _mi_meta_free(parent, subproc, memid); + return _mi_subproc_to_id(NULL); + } + theap_meta->memid = memid; + + // init subproc + mi_subproc_init(subproc,parent); + + // init main heap + mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true); + if (heap_main==NULL) { + _mi_meta_free(parent, theap_meta, theap_meta->memid); + mi_subproc_destroy(_mi_subproc_to_id(subproc)); + return _mi_subproc_to_id(NULL); + } + mi_assert_internal(subproc->heap_main == heap_main); + + // init meta theap + mi_assert_internal(parent->theap_meta!=NULL); + mi_assert_internal(parent->theap_meta->tld!=NULL); + mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); + _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); + subproc->theap_meta = theap_meta; + + return _mi_subproc_to_id(subproc); +} + + +/* ----------------------------------------------------------- + Sub process destruction +----------------------------------------------------------- */ + +// destroy all subproc resources including arena's, heap's etc. +static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock) +{ + if (subproc==NULL) return; + + // remove from the subproc list + mi_lock_maybe(&mi_subprocs_lock, acquire_subprocs_lock) { + if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; } + if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; } + else { mi_assert_internal(mi_subprocs==subproc); mi_subprocs = subproc->next; } + } + + // destroy all subproc heaps + mi_lock(&subproc->heaps_lock) { + mi_heap_t* heap = subproc->heaps; + while (heap != NULL) { + mi_heap_t* next = heap->next; + if (heap!=subproc->heap_main) { _mi_heap_force_destroy(heap, false /* don't re-acquire the heaps_lock */); } + heap = next; + } + mi_assert_internal(subproc->heap_main==NULL || subproc->heaps == subproc->heap_main); + if (subproc->heap_main!=NULL) { + _mi_thread_locals_thread_done(); // release thread locals that may have been allocated (safe as the main heap uses the fast key) + if (_mi_subproc_is_main(subproc)) { + _mi_thread_locals_done(); + } + _mi_heap_force_destroy(subproc->heap_main, false /* don't re-acquire the heaps_lock */); // no warning if destroying the main heap + } + } + + subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + + if (!_mi_subproc_is_main(subproc)) { + // merge stats back into the main subproc + _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); + } + + // remove associated arenas + _mi_arenas_unsafe_destroy_all(subproc); + + // show stats of the main process (at process end) before releasing the heaps lock + if (_mi_subproc_is_main(subproc)) { + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + } + } + + // todo: should we refcount subprocesses? + mi_lock_done(&subproc->arena_reserve_lock); + mi_lock_done(&subproc->heaps_lock); + mi_lock_done(&subproc->theap_meta_lock); + _mi_meta_free( subproc->parent, subproc, subproc->memid); + if (_mi_subproc_is_main(subproc)) { + // for the main subproc, also release the global page map + _mi_page_map_unsafe_destroy(); + } +} + +void mi_subproc_destroy(mi_subproc_id_t subproc_id) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL || subproc==&mi_process_subproc_main) return; + mi_subproc_unsafe_destroy(subproc, true /* take lock */); +} + +void _mi_subprocs_unsafe_destroy_all(void) { + mi_lock(&mi_subprocs_lock) { + mi_subproc_t* subproc = mi_subprocs; + while (subproc!=NULL) { + mi_subproc_t* next = subproc->next; + if (subproc!=&mi_process_subproc_main) { + mi_subproc_unsafe_destroy(subproc, false /* take mi_subprocs lock */); + } + subproc = next; + } + } + mi_subproc_unsafe_destroy(&mi_process_subproc_main, true /* take mi_subprocs lock */); +} + + +/* ----------------------------------------------------------- + Sub process various +----------------------------------------------------------- */ + +void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + mi_assert_internal(subproc!=NULL); + if (subproc==NULL) return; + mi_assert_internal(subproc->heap_main!=NULL); + if (subproc->heap_main==NULL) return; + mi_theap_t* theap = _mi_theap_default(); + if (mi_theap_is_initialized(theap)) { + if (theap->tld!=NULL && theap->tld->subproc != subproc) { + _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (at %p)\n", theap->tld->subproc); + } + return; + } + + // initialize this thread tld & theap + _mi_thread_init_with_heap(subproc->heap_main); +} + + +bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + bool ok = true; + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) { + ok = (*visitor)(heap, arg); + } + } + return ok; +} + + +mi_subproc_t* _mi_subproc_main_init(void) { + mi_lock_init(&mi_subprocs_lock); + mi_memid_t memid = _mi_memid_create_static(&mi_process_subproc_main,sizeof(mi_subproc_t)); + mi_process_subproc_main.memid = memid; + mi_subproc_init(&mi_process_subproc_main,NULL); + return &mi_process_subproc_main; +} + +void _mi_subproc_main_done(void) { + mi_lock_done(&mi_subprocs_lock); +} + diff --git a/system/lib/mimalloc/src/theap.c b/system/lib/mimalloc/src/theap.c index 8d5f0fb06dc9c..d9fb4309d96d3 100644 --- a/system/lib/mimalloc/src/theap.c +++ b/system/lib/mimalloc/src/theap.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_theap_default +#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "mimalloc/prim-tls.h" // _mi_theap_default #if defined(_MSC_VER) && (_MSC_VER < 1920) #pragma warning(disable:4204) // non-constant aggregate initializer @@ -23,7 +24,7 @@ typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi // Visit all pages in a theap; returns `false` if break was called. static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2) { - if (theap==NULL || theap->page_count==0) return 0; + if (theap==NULL || theap->page_count==0) return true; // visit all pages #if MI_DEBUG>1 @@ -50,19 +51,24 @@ static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, } -#if MI_DEBUG>=2 +#if MI_DEBUG>=3 static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { MI_UNUSED(arg1); MI_UNUSED(arg2); MI_UNUSED(pq); mi_assert_internal(mi_page_theap(page) == theap); + mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); + mi_assert_internal(page_theap == NULL || theap == page_theap); mi_assert_expensive(_mi_page_is_valid(page)); return true; } -#endif -#if MI_DEBUG>=3 + static bool mi_theap_is_valid(mi_theap_t* theap) { mi_assert_internal(theap!=NULL); + mi_heap_t* const heap = _mi_theap_heap_peek(theap); + mi_assert_internal(heap != NULL); + mi_theap_t* const heap_theap = _mi_heap_theap_peek(heap); // don't use mi_heap_theap as that may re-initialize the thread + mi_assert_internal(heap_theap==NULL || heap_theap == theap); mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL); for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin])); @@ -91,7 +97,7 @@ typedef enum mi_collect_e { static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { MI_UNUSED(arg2); MI_UNUSED(theap); - mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); + mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page, collect >= MI_FORCE); if (mi_page_all_free(page)) { @@ -108,7 +114,7 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag return true; // don't break } -static void mi_theap_merge_stats(mi_theap_t* theap) { +void _mi_theap_merge_stats(mi_theap_t* theap) { mi_assert_internal(mi_theap_is_initialized(theap)); mi_heap_t* const heap = _mi_theap_heap(theap); _mi_stats_merge_into(&heap->stats, &theap->stats); @@ -125,18 +131,20 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) // python/cpython#112532: we may be called from a thread that is not the owner of the theap // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id()); - // collect retired pages - _mi_theap_collect_retired(theap, force); + // collect retired pages (and full pages if theap->allow_page_abandon is false) + _mi_theap_collect_retired(theap, force); // collect all pages owned by this thread mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220 // collect arenas (this is program wide so don't force purges on abandonment of threads) //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1); - _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld); + if (collect != MI_ABANDON) { + _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld); + } // merge statistics - mi_theap_merge_stats(theap); + _mi_theap_merge_stats(theap); } void _mi_theap_collect_abandon(mi_theap_t* theap) { @@ -179,6 +187,51 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { return previous; } +#if MI_GUARDED +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + theap->guarded_sample_rate = sample_rate; + theap->guarded_sample_count = sample_rate; // count down samples + if (theap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_theap_random_next(theap); + } + theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + theap->guarded_size_min = min; + theap->guarded_size_max = (min > max ? min : max); +} + +static void mi_theap_guarded_init(mi_theap_t* theap) { + mi_theap_guarded_set_sample_rate(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_theap_guarded_set_size_bound(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); +} +#else +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); +} +static void mi_theap_guarded_init(mi_theap_t* theap) { + MI_UNUSED(theap); +} +#endif + +static void mi_theap_options_init(mi_theap_t* theap) { + theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); + theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); + theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); + theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED); +} + // todo: make order of parameters consistent (but would that break compat with CPython?) void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) { @@ -189,16 +242,15 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t)); theap->memid = memid; theap->tld = tld; // avoid reading the thread-local tld during initialization - mi_atomic_store_release(&theap->refcount,1); - mi_atomic_store_release(&theap->freed,0); - mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap); + mi_atomic_store_release(&theap->refcount,1); + mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc); mi_assert_internal(theap->stats.size == sizeof(mi_stats_t)); - - _mi_theap_options_init(theap); + mi_theap_options_init(theap); + if (theap->tld->is_in_threadpool) { // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap. // this is checked in `free.c:mi_free_try_collect_mt` - // .. but abandoning is good in this case: halve the full page retain (possibly to 0) + // .. but abandoning is good in this case: quarter the full page retain (possibly to 0) // (so blocked threads do not hold on to too much memory) if (theap->page_full_retain > 0) { theap->page_full_retain = theap->page_full_retain / 4; @@ -220,20 +272,29 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } // initialize random if heap==NULL - if (head == NULL) { // first theap in this thread? + if (head==NULL) { // first theap of the first thread? #if defined(_WIN32) && !defined(MI_SHARED_LIB) + if (tld->thread_seq==0) { _mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) - #else - _mi_random_init(&theap->random); + } + else #endif + { + _mi_random_init(&theap->random); + } } else { _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random } - theap->cookie = _mi_theap_random_next(theap) | 1; - _mi_theap_guarded_init(theap); - mi_subproc_stat_increase(_mi_subproc(),theaps,1); + // theap->cookie = _mi_theap_random_next(theap) | 1; + mi_theap_guarded_init(theap); // needs theap->random + if (!theap->is_detached) { + mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem + } + // only now set the heap member as it is used to determine if a theap is initialized + mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap); + // push on the heap's theap list mi_lock(&heap->theaps_lock) { head = heap->theaps; @@ -244,20 +305,18 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } } -mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) { mi_assert_internal(tld!=NULL); mi_assert_internal(heap!=NULL); + mi_assert_internal(tld->thread_id == MI_THREADID_DETACHED || _mi_thread_id() == tld->thread_id); + // mi_assert_internal(_mi_heap_theap_peek(heap)==NULL); // don't access thread locals as this is called on thread init + // allocate and initialize a theap mi_memid_t memid; mi_theap_t* theap; - //if (!_mi_is_heap_main(heap)) { - // theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t)); - // memid = _mi_memid_create(MI_MEM_HEAP_MAIN); - // memid.initially_zero = memid.initially_committed = true; - //} - //else + if (heap->exclusive_arena == NULL) { - theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid); + theap = (mi_theap_t*)_mi_meta_zalloc(heap->subproc, sizeof(mi_theap_t), &memid); } else { // theaps associated with a specific arena are allocated in that arena @@ -269,7 +328,14 @@ mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { _mi_error_message(ENOMEM, "unable to allocate theap meta-data\n"); return NULL; } + theap->memid = memid; + return theap; +} + +mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { + mi_theap_t* theap = _mi_theap_alloc(heap,tld); + if (theap == NULL) return NULL; _mi_theap_init(theap, heap, tld); return theap; } @@ -280,24 +346,17 @@ uintptr_t _mi_theap_random_next(mi_theap_t* theap) { static void mi_theap_free_mem(mi_theap_t* theap) { if (theap!=NULL) { - mi_subproc_stat_decrease(_mi_subproc(),theaps,1); - // free the used memory - if (theap->memid.memkind == MI_MEM_HEAP_MAIN) { // note: for now unused as it would access theap_default stats in mi_free of the current theap - mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap))); - mi_free(theap); - } - else if (theap->memid.memkind == MI_MEM_META) { - _mi_meta_free(theap, sizeof(*theap), theap->memid); - } - else { - _mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + if (!theap->is_detached) { + mi_subproc_stat_decrease(subproc,theaps,1); } + _mi_meta_free(subproc, theap, theap->memid); } } // we need to reference count theaps due to the _mi_theap_cached thread locals void _mi_theap_incref(mi_theap_t* theap) { - if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) { + if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) { mi_atomic_increment_acq_rel(&theap->refcount); } } @@ -310,184 +369,106 @@ void _mi_theap_decref(mi_theap_t* theap) { } } - -// called from `mi_theap_delete` to free the internal theap resources. -bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) { - mi_assert(theap != NULL); - if (theap==NULL) return true; - - // ensure only one thread actually frees the theap - const size_t freed = mi_atomic_exchange_acq_rel( &theap->freed, 1 ); - if (freed!=0) { - // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock) - return false; - } - else { - // merge stats to the owning heap - mi_heap_t* const heap = _mi_theap_heap(theap); - _mi_stats_merge_into(&heap->stats, &theap->stats); - - // remove ourselves from the heap theaps list - mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) { - if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } - if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } - else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } - theap->hnext = theap->hprev = NULL; +// Thread termination and heap delete/destroy might run concurrently +// and we need to ensure we free the memory correctly. A heap or tld +// will first "detach" its theaps so it has a list with theaps that are +// no longer shared, and only then free's the theaps in that list. +// To detach we need to hold both the `heap->theaps_lock` and the `tld->theaps_lock`. +// Due to lock-inversion we need to use `mi_lock_try_acquire` and if that fails +// we back-off, release the outer lock, and try again until we succeed. + +// Remove the theaps in this heap from any thread local tld lists. +void _mi_heap_detach_theaps( mi_heap_t* heap ) { + bool all_detached; + do { + all_detached = true; + mi_lock(&heap->theaps_lock) { + mi_theap_t* theap = heap->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->hnext; + mi_tld_t* tld = theap->tld; + if (tld != NULL) { + if (mi_lock_try_acquire(&tld->theaps_lock)) { + // remove the theap from the tld theaps list + if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } + if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } + else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; } + theap->tnext = theap->tprev = NULL; + theap->tld = NULL; + mi_lock_release(&tld->theaps_lock); + } + else { + all_detached = false; + } + } + theap = next; + } } - - // remove ourselves from the thread local theaps list - mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) { - if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } - if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } - else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; } - theap->tnext = theap->tprev = NULL; + if (!all_detached) { + mi_subproc_stat_counter_increase(heap->subproc,heaps_delete_wait,1); + _mi_prim_thread_yield(); } - - // Set heap to NULL only after we are removed from the thread local theaps list since - // we may concurrently traverse it to collect (in `init.c:mi_thread_theaps_done`) - // (We need to set it to NULL to avoid an ABA problem where the _mi_theap_cached - // has a heap address that is reused for a newly allocated heap.) - mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); - theap->tld = NULL; - _mi_theap_decref(theap); - return true; - } -} - - -/* ----------------------------------------------------------- - Heap destroy ------------------------------------------------------------ */ -/* - -// zero out the page queues -static void mi_theap_reset_pages(mi_theap_t* theap) { - mi_assert_internal(theap != NULL); - mi_assert_internal(mi_theap_is_initialized(theap)); - // TODO: copy full empty theap instead? - _mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct)); - _mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages)); - // theap->thread_delayed_free = NULL; - theap->page_count = 0; -} - -static bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - MI_UNUSED(arg1); - MI_UNUSED(arg2); - MI_UNUSED(pq); - - // ensure no more thread_delayed_free will be added - //_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); - - // stats - const size_t bsize = mi_page_block_size(page); - if (bsize > MI_LARGE_MAX_OBJ_SIZE) { - mi_theap_stat_decrease(theap, malloc_huge, bsize); - } - #if (MI_STAT>0) - _mi_page_free_collect(page, false); // update used count - const size_t inuse = page->used; - if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { - mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse); - #if (MI_STAT>1) - mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse); - #endif - } - // mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse); // todo: off for aligned blocks... - #endif - - /// pretend it is all free now - mi_assert_internal(mi_page_thread_free(page) == NULL); - page->used = 0; - - // and free the page - // mi_page_free(page,false); - page->next = NULL; - page->prev = NULL; - mi_page_set_theap(page, NULL); - _mi_arenas_page_free(page, theap); - - return true; // keep going -} - -void _mi_theap_destroy_pages(mi_theap_t* theap) { - mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL); - mi_theap_reset_pages(theap); -} - -#if MI_TRACK_HEAP_DESTROY -static bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) { - MI_UNUSED(theap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size); - mi_track_free_size(block,mi_usable_size(block)); - return true; -} -#endif - -void mi_theap_destroy(mi_theap_t* theap) { - mi_assert(theap != NULL); - mi_assert(mi_theap_is_initialized(theap)); - mi_assert(!theap->allow_page_reclaim); - mi_assert(!theap->allow_page_abandon); - mi_assert_expensive(mi_theap_is_valid(theap)); - if (theap==NULL || !mi_theap_is_initialized(theap)) return; - #if MI_GUARDED - // _mi_warning_message("'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\n", theap); - mi_theap_delete(theap); - return; - #else - if (theap->allow_page_reclaim) { - _mi_warning_message("'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\n", theap); - // don't free in case it may contain reclaimed pages, - mi_theap_delete(theap); - } - else { - // track all blocks as freed - #if MI_TRACK_HEAP_DESTROY - mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL); - #endif - // free all pages - _mi_theap_destroy_pages(theap); - mi_theap_free(theap,true); - } - #endif -} - -// forcefully destroy all theaps in the current thread -void _mi_theap_unsafe_destroy_all(mi_theap_t* theap) { - mi_assert_internal(theap != NULL); - if (theap == NULL) return; - mi_theap_t* curr = theap->tld->theaps; - while (curr != NULL) { - mi_theap_t* next = curr->next; - if (!curr->allow_page_reclaim) { - mi_theap_destroy(curr); + } while (!all_detached); +} + +// Remove the theaps in this thread from the heaps that own them. +void _mi_tld_detach_theaps( mi_tld_t* tld ) { + bool all_detached; + do { + all_detached = true; + mi_lock(&tld->theaps_lock) { + mi_theap_t* theap = tld->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + mi_assert_internal(theap->page_count==0); + mi_heap_t* heap = _mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration + if (heap != NULL) { + if (mi_lock_try_acquire(&heap->theaps_lock)) { + // merge stats into the owning heap stats + _mi_stats_merge_into(&heap->stats, &theap->stats); + // remove the theap from the heap list + if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } + if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } + else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } + theap->hnext = theap->hprev = NULL; + // and set `heap` to NULL + mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); + mi_lock_release(&heap->theaps_lock); + } + else { + all_detached = false; + } + } + theap = next; + } } - else { - _mi_theap_destroy_pages(curr); + if (!all_detached) { + mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); + _mi_prim_thread_yield(); } - curr = next; - } + } while (!all_detached); } -*/ + + /* ----------------------------------------------------------- - Safe Heap delete + Safe theap delete ----------------------------------------------------------- */ // Safe delete a theap without freeing any still allocated blocks in that theap. -void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock) -{ - mi_assert(theap != NULL); - mi_assert(mi_theap_is_initialized(theap)); - mi_assert_expensive(mi_theap_is_valid(theap)); - if (theap==NULL || !mi_theap_is_initialized(theap)) return; +// void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock) +// { +// mi_assert(theap != NULL); +// mi_assert(mi_theap_is_initialized(theap)); +// mi_assert_expensive(mi_theap_is_valid(theap)); +// if (theap==NULL || !mi_theap_is_initialized(theap)) return; - // abandon all pages - _mi_theap_collect_abandon(theap); +// // abandon all pages +// _mi_theap_collect_abandon(theap); - mi_assert_internal(theap->page_count==0); - _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock); -} +// mi_assert_internal(theap->page_count==0); +// _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock); +// } @@ -541,7 +522,7 @@ bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) { // reinit direct pages (as we may be in a different process) mi_assert_internal(theap->page_count == 0); for (size_t i = 0; i < MI_PAGES_DIRECT; i++) { - theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty; + theap->pages_free_direct[i] = _mi_page_empty_get(); } // push on the thread local theaps list @@ -686,11 +667,11 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi return true; } -bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) { - mi_heap_area_t area; - _mi_heap_area_init(&area, page); - return _mi_theap_area_visit_blocks(&area, page, visitor, arg); -} +// bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) { +// mi_heap_area_t area; +// _mi_heap_area_init(&area, page); +// return _mi_theap_area_visit_blocks(&area, page, visitor, arg); +// } // Separate struct to keep `mi_page_t` out of the public interface diff --git a/system/lib/mimalloc/src/threadlocal.c b/system/lib/mimalloc/src/threadlocal.c index eea3329bfaad9..355498ceeef6f 100644 --- a/system/lib/mimalloc/src/threadlocal.c +++ b/system/lib/mimalloc/src/threadlocal.c @@ -6,9 +6,9 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ /* ---------------------------------------------------------------------------- -Implement dynamic thread local variables (for heap's). -Unlike most OS native implementations there is no limit on the number -that can be allocated. + Implement dynamic thread local variables (used by heap's for their theap's). + Unlike most OS native implementations there is no limit on the number + that can be allocated. -----------------------------------------------------------------------------*/ #include "mimalloc.h" @@ -27,12 +27,41 @@ typedef struct mi_tls_slot_s { typedef struct mi_thread_locals_s { size_t count; + mi_memid_t memid; mi_tls_slot_t slots[1]; } mi_thread_locals_t; -static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} }; +static mi_thread_locals_t mi_thread_locals_empty = mi_init_struct_zero; -mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty; // always point to a valid `mi_thread_locals_t` + +/* ----------------------------------------------------------- + We have 2 thread local variable which we implement with either + a C thread local declaration or using pthread keys. + - mi_thread_locals: points to an array of thread locals for most keys + - mi_slot_fast: a single dedicated thread local for slightly faster access. (used for the main heap's theap) +----------------------------------------------------------- */ + +#if MI_TLS_MODEL_PTHREADS || defined(__APPLE__) // macOS has fast pthreads +// Use pthreads +#define mi_define_thread_local(tp,name,initval) \ + static pthread_key_t __##name##_key = MI_PTHREAD_KEY_INVALID; \ + static inline tp name##_peek(void) { return (tp)mi_pthread_key_get(__##name##_key); } \ + static inline tp name##_get(void) { tp result = name##_peek(); return (result!=NULL ? result : initval); } \ + static inline bool name##_set(tp val) { return mi_pthread_key_set(&__##name##_key,val); } \ + static inline void name##_delete(void) { mi_pthread_key_delete(&__##name##_key); } + +#else +// Direct thread locals +#define mi_define_thread_local(tp,name,initval) \ + static mi_decl_thread tp __##name = initval; \ + static inline tp name##_peek(void) { return __##name; } \ + static inline tp name##_get(void) { tp result = __##name; return (result!=NULL ? result : initval); } \ + static inline bool name##_set(tp val) { __##name = val; return true; } \ + static inline void name##_delete(void) { } +#endif + +mi_define_thread_local(mi_thread_locals_t*, mi_thread_locals, &mi_thread_locals_empty) +mi_define_thread_local(void*, mi_slot_fast, NULL) /* ----------------------------------------------------------- @@ -42,15 +71,18 @@ mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty; a value, we also set the version of the key. ----------------------------------------------------------- */ -#if MI_SIZE_BITS < 64 -#define MI_TLS_IDX_BITS (MI_SIZE_BITS/2) // half for the index, half for the version +#if MI_SIZE_BITS >= 64 +#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) /* 16 bits for the index, 48 bits for the version */ +#elif MI_SIZE_BITS >= 32 +#define MI_TLS_IDX_BITS (12) /* 12 bits for index, 20 for the version? */ #else -#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) // 16 bits for the index, 48 bits for the version +#error not enough bits for the version for thread locals #endif #define MI_TLS_IDX_MASK ((MI_ZU(1)<count; size_t count; if (count_old==0) { tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty count = 16; // start with 16 slots - } + } else if (count_old >= 1024) { count = count_old + 1024; // at some point increase linearly } else { count = 2*count_old; // and double initially } - if (count <= least_idx) { + if (count <= least_idx) { count = least_idx + 1; } if (count > MI_TLS_IDX_MAX) { return NULL; } // too large - mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t)); + // allocate as meta (for secure mode) + // we could also allocate on the main heap; this is recursion safe as that uses the fast local key + mi_memid_t memid = (tls_old==NULL ? _mi_memid_none() : tls_old->memid); + mi_thread_locals_t* tls = (mi_thread_locals_t*)_mi_meta_rezalloc(_mi_subproc(), tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t), &memid); if mi_unlikely(tls==NULL) return NULL; + tls->memid = memid; tls->count = count; - mi_thread_locals = tls; + mi_thread_locals_set(tls); return tls; } static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) { if (val==NULL) return true; - const size_t idx = mi_key_index(key); + const size_t idx = mi_key_index(key); mi_thread_locals_t* tls = mi_thread_locals_expand(idx); - if (tls==NULL) return false; - mi_assert_internal(tls == mi_thread_locals); + if (tls==NULL) { + _mi_error_message(EFAULT,"unable to allocate thread local variables\n"); + return false; + } + mi_assert_internal(tls == mi_thread_locals_get()); mi_assert_internal(idx < tls->count); tls->slots[idx].value = val; tls->slots[idx].version = mi_key_version(key); @@ -108,8 +147,8 @@ static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, // set a tls slot; returns `true` if successful. // Can return `false` if we could not reallocate the slots array. -bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { - mi_thread_locals_t* tls = mi_thread_locals; +static mi_decl_noinline bool mi_thread_local_set_regular( mi_thread_local_t key, void* val ) { + mi_thread_locals_t* tls = mi_thread_locals_get(); mi_assert_internal(tls!=NULL); mi_assert_internal(key!=0); const size_t idx = mi_key_index(key); @@ -123,25 +162,54 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { } } +bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { + mi_assert_internal(key!=0); + if (key == mi_thread_local_key_fast) { + return mi_slot_fast_set(val); + } + else { + return mi_thread_local_set_regular(key,val); + } +} + // get a tls slot value -void* _mi_thread_local_get( mi_thread_local_t key ) { - const mi_thread_locals_t* const tls = mi_thread_locals; - mi_assert_internal(tls!=NULL); +static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) { mi_assert_internal(key!=0); + const mi_thread_locals_t* const tls = mi_thread_locals_peek(); + if mi_unlikely(tls==NULL) { + // this can happen if a thread local is accessed after the thread local has been freed + // from mi_thread_done or mi_process_done. + // todo: can we remove this check? now we can still call this from process done when stats are printed (which calls mi_heap_theap_peek) + return NULL; + } const size_t idx = mi_key_index(key); if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) { return tls->slots[idx].value; } else { - return NULL; + return NULL; + } +} + +// get a thread local value +void* _mi_thread_local_get( mi_thread_local_t key ) { + mi_assert_internal(key!=0); + if mi_likely(key == mi_thread_local_key_fast) { + return mi_slot_fast_get(); + } + else { + return mi_thread_local_get_regular(key); } } void _mi_thread_locals_thread_done(void) { - mi_thread_locals_t* const tls = mi_thread_locals; + mi_thread_locals_t* const tls = mi_thread_locals_peek(); if (tls!=NULL && tls->count > 0) { - mi_free(tls); - mi_thread_locals = &mi_thread_locals_empty; + _mi_meta_free(_mi_subproc(), tls, tls->memid); + mi_thread_locals_set(NULL); + } + if (mi_slot_fast_peek() != NULL) { + mi_slot_fast_set(NULL); } } @@ -152,6 +220,7 @@ Create and free fresh TLS key's static mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits static mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use) +static mi_memid_t mi_thread_locals_memid; // provenance of mi_thread_locals_free static size_t mi_thread_locals_version; // version to be able to reuse slots safely void _mi_thread_locals_init(void) { @@ -161,9 +230,13 @@ void _mi_thread_locals_init(void) { void _mi_thread_locals_done(void) { mi_lock(&mi_thread_locals_lock) { mi_bitmap_t* const slots = mi_thread_locals_free; - mi_free(slots); + if (slots!=NULL) { + _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); + } } mi_lock_done(&mi_thread_locals_lock); + mi_thread_locals_delete(); + mi_slot_fast_delete(); } // strange signature but allows us to reuse the arena code for claiming free pages @@ -173,7 +246,7 @@ static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, b return true; } -// When we claim a free slot, we increase the global version counter +// When we claim a free slot, we increase the global version counter // (so if we reuse a slot it will be returning NULL initially when a thread tries to get it) static mi_thread_local_t mi_thread_local_claim(void) { size_t idx = 0; @@ -194,16 +267,21 @@ static bool mi_thread_local_create_expand(void) { const size_t newcount = 1024 + oldcount; if (newcount > MI_TLS_IDX_MAX) { return false; } const size_t newsize = mi_bitmap_size( newcount, NULL ); - mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE); + // mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE); + mi_memid_t memid; + mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc_aligned(_mi_subproc_main(), newsize, MI_BCHUNK_SIZE, &memid); // always allocate thread locals in the main subprocess + mi_assert_internal(_mi_is_aligned(newslots,MI_BCHUNK_SIZE)); if (newslots==NULL) { return false; } if (slots!=NULL) { // copy over the previous bitmap - _mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL)); - mi_free(slots); + const size_t oldsize = mi_bitmap_size(oldcount,NULL); + _mi_memcpy_aligned(newslots, slots, oldsize); + _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); } mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */); mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */ mi_thread_locals_free = newslots; + mi_thread_locals_memid = memid; return true; } @@ -219,6 +297,8 @@ mi_thread_local_t _mi_thread_local_create(void) { } } } + mi_assert_internal(key!=0); + mi_assert_internal(key!=mi_thread_local_key_fast); return key; } diff --git a/tools/system_libs.py b/tools/system_libs.py index bbdf3944b8983..a131a025d0ed3 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1907,6 +1907,7 @@ class libmimalloc(MTLibrary): excludes={'alloc-override.c', 'free.c', 'page-queue.c', 'static.c'}, ) src_files += [utils.path_from_root('system/lib/mimalloc/src/prim/prim.c')] + src_files += [utils.path_from_root('system/lib/mimalloc/src/prim/prim-tls.c')] src_files += [utils.path_from_root('system/lib/emmalloc.c')] # Include sbrk.c in libc, it uses tracing and libc itself doesn't have a tracing variant. src_files += [utils.path_from_root('system/lib/libc/sbrk.c')] From 4b56f61e20e6fdbe6b4d04ba1d3e6fd9499110e8 Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Fri, 4 Sep 2026 13:06:09 +0200 Subject: [PATCH 3/5] mimalloc: add Emscripten-specific changes This reverts commit d70353c0. --- system/lib/mimalloc/src/alloc-override.c | 19 +++++++++++++++++-- .../lib/mimalloc/src/prim/emscripten/prim.c | 12 ++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/system/lib/mimalloc/src/alloc-override.c b/system/lib/mimalloc/src/alloc-override.c index 04da5f2935d59..df1a9d6863d22 100644 --- a/system/lib/mimalloc/src/alloc-override.c +++ b/system/lib/mimalloc/src/alloc-override.c @@ -32,7 +32,8 @@ typedef void* mi_nothrow_t; #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); #else - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); + // XXX EMSCRIPTEN: Add "weak" + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), weak)); #endif #define MI_FORWARD1(fun,x) MI_FORWARD(fun) #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) @@ -339,16 +340,18 @@ mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_malloc_size,p) - #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) && !defined(__EMSCRIPTEN__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_malloc_usable_size,p) #else size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_malloc_usable_size,p) #endif // No forwarding here due to aliasing/name mangling issues + mi_decl_weak // XXX EMSCRIPTEN void* valloc(size_t size) { return mi_valloc(size); } void vfree(void* p) { mi_free(p); } size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } + mi_decl_weak // XXX EMSCRIPTEN int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } // `aligned_alloc` is only available when __USE_ISOC11 is defined. @@ -359,6 +362,7 @@ mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. #if !defined(__GLIBC__) || __USE_ISOC11 + mi_decl_weak // XXX EMSCRIPTEN void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #endif #endif @@ -366,10 +370,12 @@ mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { // no forwarding here due to aliasing/name mangling issues void cfree(void* p) { mi_free(p); } void* pvalloc(size_t size) { return mi_pvalloc(size); } +mi_decl_weak // XXX EMSCRIPTEN void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } #if !defined(_WIN32) void* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); } #endif +mi_decl_weak // XXX EMSCRIPTEN void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } // some systems define reallocarr so mark it as a weak symbol (#751) mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } @@ -380,8 +386,17 @@ mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_r void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size) void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) void __libc_free(void* p) MI_FORWARD0(mi_free, p) + mi_decl_weak // XXX EMSCRIPTEN void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +#ifdef __EMSCRIPTEN__ // emscripten adds some more on top of WASI + void* emscripten_builtin_malloc(size_t size) MI_FORWARD1(mi_malloc, size) + void* emscripten_builtin_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size) + void emscripten_builtin_free(void* p) MI_FORWARD0(mi_free, p) + void* emscripten_builtin_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } + void* emscripten_builtin_calloc(size_t nmemb, size_t size) MI_FORWARD2(mi_calloc, nmemb, size) +#endif + #elif defined(__linux__) // forward __libc interface (needed for glibc-based and musl-based Linux distributions) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) diff --git a/system/lib/mimalloc/src/prim/emscripten/prim.c b/system/lib/mimalloc/src/prim/emscripten/prim.c index 992965d56f23b..4386456baa7e3 100644 --- a/system/lib/mimalloc/src/prim/emscripten/prim.c +++ b/system/lib/mimalloc/src/prim/emscripten/prim.c @@ -74,7 +74,7 @@ extern void* emmalloc_memalign(size_t alignment, size_t size); // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { - MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); + MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; // todo: Track the highest address ever seen; first uses of it are zeroes. // That assumes no one else uses sbrk but us (they could go up, @@ -85,12 +85,8 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm if (try_alignment < MIN_EMMALLOC_ALIGN) { try_alignment = MIN_EMMALLOC_ALIGN; } - void* p = emmalloc_memalign(try_alignment, size); - *addr = p; - if (p == 0) { - return ENOMEM; - } - return 0; + *addr = emmalloc_memalign(try_alignment, size); + return (*addr != NULL ? 0 : ENOMEM); } @@ -177,7 +173,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) #include void _mi_prim_out_stderr( const char* msg) { - emscripten_console_error(msg); + emscripten_err(msg); } From da347f6215e59505edbf7afd8774e50f326cc46b Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Fri, 4 Sep 2026 13:23:14 +0200 Subject: [PATCH 4/5] Rebaseline `other.test_malloc_size_mimalloc_O2` codesize expectations Details: AssertionError: 196838 not less than 196000 --- test/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_other.py b/test/test_other.py index 5b113141e2d0b..3eb1c2d5e0099 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -7761,7 +7761,7 @@ def test_dlmalloc_modes(self): 'mimalloc': (['-sMALLOC=mimalloc'], 260000), 'emmalloc_O2': (['-sMALLOC=emmalloc', '-O2'], 131000), 'dlmalloc_O2': (['-sMALLOC=dlmalloc', '-O2'], 137000), - 'mimalloc_O2': (['-sMALLOC=mimalloc', '-O2'], 196000), + 'mimalloc_O2': (['-sMALLOC=mimalloc', '-O2'], 197000), }) # This test verifies the output code size of the different -sMALLOC= modes. def test_malloc_size(self, args, max_size): From ac97f4d4bb3ec890125d2d70ae761b490ac98464 Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Fri, 4 Sep 2026 13:27:43 +0200 Subject: [PATCH 5/5] Update ChangeLog.md --- ChangeLog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.md b/ChangeLog.md index 2d9c66fa35945..74ede3362cf3d 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -23,6 +23,7 @@ See docs/process.md for more on how version tagging works. - The SDL3 port is no longer considered experimental, and the compiler diagnostic warning has been removed. (#27646) - `WASM=0` and `WASM=2` (wasm2js) were marked as deprecated. (See #27608) +- mimalloc was updated to 3.5.1. (#27662) 6.0.9 - 09/01/26 ----------------