From 384e553d62a8936114641d8a434376deda232dcf Mon Sep 17 00:00:00 2001 From: Rui Ueyama Date: Mon, 20 Jan 2025 20:14:53 +0900 Subject: [PATCH] Upgrade bundled mimalloc to 2.1.9 --- third-party/mimalloc/.gitignore | 1 - third-party/mimalloc/CMakeLists.txt | 153 +++- third-party/mimalloc/LICENSE | 2 +- third-party/mimalloc/azure-pipelines.yml | 208 ++++-- .../mimalloc/bin/mimalloc-redirect-arm64.dll | Bin 0 -> 55296 bytes .../mimalloc/bin/mimalloc-redirect-arm64.lib | Bin 0 -> 2976 bytes .../bin/mimalloc-redirect-arm64ec.dll | Bin 0 -> 96768 bytes .../bin/mimalloc-redirect-arm64ec.lib | Bin 0 -> 3308 bytes .../mimalloc/bin/mimalloc-redirect.dll | Bin 68096 -> 53760 bytes .../mimalloc/bin/mimalloc-redirect.lib | Bin 2874 -> 2874 bytes .../mimalloc/bin/mimalloc-redirect32.dll | Bin 41984 -> 35328 bytes .../mimalloc/bin/mimalloc-redirect32.lib | Bin 2928 -> 2928 bytes third-party/mimalloc/bin/minject-arm64.exe | Bin 0 -> 20992 bytes third-party/mimalloc/bin/minject.exe | Bin 22016 -> 22016 bytes third-party/mimalloc/bin/minject32.exe | Bin 18944 -> 18944 bytes third-party/mimalloc/bin/readme.md | 79 +- .../cmake/mimalloc-config-version.cmake | 2 +- third-party/mimalloc/doc/doxyfile | 696 ++++++++++++------ third-party/mimalloc/doc/mimalloc-doc.h | 413 ++++++++--- third-party/mimalloc/doc/mimalloc-doxygen.css | 11 + .../mimalloc/docker/alpine-arm32v7/Dockerfile | 6 +- third-party/mimalloc/docker/alpine/Dockerfile | 2 +- .../mimalloc/docker/manylinux-x64/Dockerfile | 2 +- .../ide/vs2017/mimalloc-override-test.vcxproj | 190 ----- .../ide/vs2017/mimalloc-override.vcxproj | 260 ------- .../ide/vs2017/mimalloc-test-stress.vcxproj | 159 ---- .../mimalloc/ide/vs2017/mimalloc-test.vcxproj | 158 ---- third-party/mimalloc/ide/vs2017/mimalloc.sln | 71 -- .../mimalloc/ide/vs2017/mimalloc.vcxproj | 260 ------- .../ide/vs2019/mimalloc-override-test.vcxproj | 190 ----- .../ide/vs2019/mimalloc-override.vcxproj | 260 ------- .../ide/vs2019/mimalloc-test-api.vcxproj | 155 ---- .../ide/vs2019/mimalloc-test-stress.vcxproj | 159 ---- .../mimalloc/ide/vs2019/mimalloc-test.vcxproj | 158 ---- third-party/mimalloc/ide/vs2019/mimalloc.sln | 81 -- .../mimalloc/ide/vs2019/mimalloc.vcxproj | 258 ------- ...{mimalloc.vcxproj => mimalloc-lib.vcxproj} | 236 +++++- .../ide/vs2022/mimalloc-lib.vcxproj.filters | 105 +++ ....vcxproj => mimalloc-override-dll.vcxproj} | 240 +++++- .../mimalloc-override-dll.vcxproj.filters | 113 +++ .../ide/vs2022/mimalloc-override-test.vcxproj | 168 ++++- .../ide/vs2022/mimalloc-test-api.vcxproj | 132 +++- .../ide/vs2022/mimalloc-test-stress.vcxproj | 134 +++- .../mimalloc/ide/vs2022/mimalloc-test.vcxproj | 138 +++- third-party/mimalloc/ide/vs2022/mimalloc.sln | 214 ++++-- third-party/mimalloc/include/mimalloc.h | 44 +- .../mimalloc/include/mimalloc/atomic.h | 170 ++++- .../mimalloc/include/mimalloc/internal.h | 381 ++++++---- third-party/mimalloc/include/mimalloc/prim.h | 86 ++- third-party/mimalloc/include/mimalloc/track.h | 8 +- third-party/mimalloc/include/mimalloc/types.h | 82 ++- third-party/mimalloc/readme.md | 223 ++++-- third-party/mimalloc/src/alloc-aligned.c | 79 +- third-party/mimalloc/src/alloc-override.c | 6 +- third-party/mimalloc/src/alloc.c | 119 ++- third-party/mimalloc/src/arena-abandon.c | 346 +++++++++ third-party/mimalloc/src/arena.c | 443 +++++------ third-party/mimalloc/src/bitmap.c | 12 +- third-party/mimalloc/src/bitmap.h | 8 +- third-party/mimalloc/src/free.c | 73 +- third-party/mimalloc/src/heap.c | 192 +++-- third-party/mimalloc/src/init.c | 258 +++---- third-party/mimalloc/src/libc.c | 20 +- third-party/mimalloc/src/options.c | 137 +++- third-party/mimalloc/src/os.c | 203 ++--- third-party/mimalloc/src/page-queue.c | 67 +- third-party/mimalloc/src/page.c | 112 ++- .../mimalloc/src/prim/emscripten/prim.c | 6 +- .../src/prim/osx/alloc-override-zone.c | 4 +- third-party/mimalloc/src/prim/prim.c | 49 ++ third-party/mimalloc/src/prim/unix/prim.c | 40 +- third-party/mimalloc/src/prim/wasi/prim.c | 13 +- third-party/mimalloc/src/prim/windows/prim.c | 262 +++++-- third-party/mimalloc/src/segment-map.c | 169 ++--- third-party/mimalloc/src/segment.c | 332 +++++++-- third-party/mimalloc/src/static.c | 2 +- third-party/mimalloc/src/stats.c | 33 +- .../mimalloc/test/main-override-static.c | 156 +++- third-party/mimalloc/test/main-override.cpp | 99 ++- third-party/mimalloc/test/test-api-fill.c | 2 +- third-party/mimalloc/test/test-api.c | 13 +- third-party/mimalloc/test/test-stress.c | 81 +- 82 files changed, 5584 insertions(+), 4160 deletions(-) create mode 100644 third-party/mimalloc/bin/mimalloc-redirect-arm64.dll create mode 100644 third-party/mimalloc/bin/mimalloc-redirect-arm64.lib create mode 100644 third-party/mimalloc/bin/mimalloc-redirect-arm64ec.dll create mode 100644 third-party/mimalloc/bin/mimalloc-redirect-arm64ec.lib create mode 100644 third-party/mimalloc/bin/minject-arm64.exe delete mode 100644 third-party/mimalloc/ide/vs2017/mimalloc-override-test.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2017/mimalloc-override.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2017/mimalloc-test-stress.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2017/mimalloc-test.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2017/mimalloc.sln delete mode 100644 third-party/mimalloc/ide/vs2017/mimalloc.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc-override-test.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc-override.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc-test-api.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc-test-stress.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc-test.vcxproj delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc.sln delete mode 100644 third-party/mimalloc/ide/vs2019/mimalloc.vcxproj rename third-party/mimalloc/ide/vs2022/{mimalloc.vcxproj => mimalloc-lib.vcxproj} (51%) create mode 100644 third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj.filters rename third-party/mimalloc/ide/vs2022/{mimalloc-override.vcxproj => mimalloc-override-dll.vcxproj} (51%) create mode 100644 third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj.filters create mode 100644 third-party/mimalloc/src/arena-abandon.c diff --git a/third-party/mimalloc/.gitignore b/third-party/mimalloc/.gitignore index df1d58eb2e..b2439f94b2 100644 --- a/third-party/mimalloc/.gitignore +++ b/third-party/mimalloc/.gitignore @@ -1,7 +1,6 @@ ide/vs20??/*.db ide/vs20??/*.opendb ide/vs20??/*.user -ide/vs20??/*.vcxproj.filters ide/vs20??/.vs ide/vs20??/VTune* out/ diff --git a/third-party/mimalloc/CMakeLists.txt b/third-party/mimalloc/CMakeLists.txt index bcfe91d867..283b0f571b 100644 --- a/third-party/mimalloc/CMakeLists.txt +++ b/third-party/mimalloc/CMakeLists.txt @@ -7,18 +7,20 @@ set(CMAKE_CXX_STANDARD 17) option(MI_SECURE "Use full security mitigations (like guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) option(MI_DEBUG_FULL "Use full internal heap invariant checking in DEBUG mode (expensive)" OFF) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) -option(MI_OVERRIDE "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON) +option(MI_OVERRIDE "Override the standard malloc interface (i.e. define entry points for 'malloc', 'free', etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) option(MI_TRACK_ETW "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) +option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for arm64: '-march=armv8.1-a' (2016))" ON) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_WIN_REDIRECT "Use redirection module ('mimalloc-redirect') on Windows if compiling mimalloc as a DLL" ON) -option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) +option(MI_WIN_USE_FIXED_TLS "Use a fixed TLS slot on Windows to avoid extra tests in the malloc fast path" OFF) +option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix)" OFF) option(MI_LIBC_MUSL "Set this when linking with musl libc" OFF) option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) @@ -26,12 +28,15 @@ option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) +option(MI_GUARDED "Build with guard pages behind certain object allocations (implies MI_NO_PADDING=ON)" OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) option(MI_NO_THP "Disable transparent huge pages support on Linux/Android for the mimalloc process only" OFF) +option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") # deprecated options +option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination (deprecated)" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) @@ -61,30 +66,39 @@ set(mi_sources set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build set(mi_cflags_dynamic "") # extra flags for a shared-object library build -set(mi_defines "") set(mi_libraries "") +if(MI_EXTRA_CPPDEFS) + set(mi_defines ${MI_EXTRA_CPPDEFS}) +else() + set(mi_defines "") +endif() + # ----------------------------------------------------------------------------- -# Convenience: set default build type depending on the build directory +# Convenience: set default build type and compiler depending on the build directory # ----------------------------------------------------------------------------- message(STATUS "") if (NOT CMAKE_BUILD_TYPE) - if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$" OR MI_DEBUG_FULL) - message(STATUS "No build type selected, default to: Debug") + if ("${CMAKE_BINARY_DIR}" MATCHES ".*((D|d)ebug|asan|tsan|ubsan|valgrind)$" OR MI_DEBUG_FULL) + message(STATUS "No build type selected, default to 'Debug'") set(CMAKE_BUILD_TYPE "Debug") else() - message(STATUS "No build type selected, default to: Release") + message(STATUS "No build type selected, default to 'Release'") set(CMAKE_BUILD_TYPE "Release") endif() endif() +if (CMAKE_GENERATOR MATCHES "^Visual Studio.*$") + message(STATUS "Note: when building with Visual Studio the build type is specified when building.") + message(STATUS "For example: 'cmake --build . --config=Release") +endif() + if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") message(STATUS "Default to secure build") set(MI_SECURE "ON") endif() - # ----------------------------------------------------------------------------- # Process options # ----------------------------------------------------------------------------- @@ -104,6 +118,14 @@ if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel") set(MI_USE_CXX "ON") endif() +if(CMAKE_BUILD_TYPE MATCHES "Release|RelWithDebInfo") + if (NOT MI_OPT_ARCH) + message(STATUS "Architecture specific optimizations are disabled (MI_OPT_ARCH=OFF)") + endif() +else() + set(MI_OPT_ARCH OFF) +endif() + if(MI_OVERRIDE) message(STATUS "Override standard malloc (MI_OVERRIDE=ON)") if(APPLE) @@ -131,12 +153,6 @@ if(MI_OVERRIDE) endif() if(WIN32) - if (MI_WIN_REDIRECT) - if (MSVC_C_ARCHITECTURE_ID MATCHES "ARM") - message(STATUS "Cannot use redirection on Windows ARM (MI_WIN_REDIRECT=OFF)") - set(MI_WIN_REDIRECT OFF) - endif() - endif() if (NOT MI_WIN_REDIRECT) # use a negative define for backward compatibility list(APPEND mi_defines MI_WIN_NOREDIRECT=1) @@ -152,8 +168,8 @@ if(MI_TRACK_VALGRIND) CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH) if (NOT MI_HAS_VALGRINDH) set(MI_TRACK_VALGRIND OFF) - message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first") - message(STATUS "Compile **without** Valgrind support (MI_TRACK_VALGRIND=OFF)") + message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first?") + message(STATUS "Disabling Valgrind support (MI_TRACK_VALGRIND=OFF)") else() message(STATUS "Compile with Valgrind support (MI_TRACK_VALGRIND=ON)") list(APPEND mi_defines MI_TRACK_VALGRIND=1) @@ -199,6 +215,15 @@ if(MI_TRACK_ETW) endif() endif() +if(MI_GUARDED) + message(STATUS "Compile guard pages behind certain object allocations (MI_GUARDED=ON)") + list(APPEND mi_defines MI_GUARDED=1) + if(NOT MI_NO_PADDING) + message(STATUS " Disabling padding due to guard pages (MI_NO_PADDING=ON)") + set(MI_NO_PADDING ON) + endif() +endif() + if(MI_SEE_ASM) message(STATUS "Generate assembly listings (MI_SEE_ASM=ON)") list(APPEND mi_cflags -save-temps) @@ -258,6 +283,7 @@ if(MI_DEBUG_UBSAN) if(CMAKE_BUILD_TYPE MATCHES "Debug") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Build with undefined-behavior sanitizer (MI_DEBUG_UBSAN=ON)") + list(APPEND mi_defines MI_UBSAN=1) list(APPEND mi_cflags -fsanitize=undefined -g -fno-sanitize-recover=undefined) list(APPEND mi_libraries -fsanitize=undefined) if (NOT MI_USE_CXX) @@ -296,6 +322,48 @@ if(MI_LIBC_MUSL) list(APPEND mi_defines MI_LIBC_MUSL=1) endif() +if(MI_WIN_USE_FLS) + message(STATUS "Use the Fiber API to detect thread termination (deprecated) (MI_WIN_USE_FLS=ON)") + list(APPEND mi_defines MI_WIN_USE_FLS=1) +endif() + +if(MI_WIN_USE_FIXED_TLS) + message(STATUS "Use fixed TLS slot on Windows to avoid extra tests in the malloc fast path (MI_WIN_USE_FIXED_TLS=ON)") + list(APPEND mi_defines MI_WIN_USE_FIXED_TLS=1) +endif() + +# Determine architecture +set(MI_OPT_ARCH_FLAGS "") +set(MI_ARCH "unknown") +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86|i[3456]86)$" OR CMAKE_GENERATOR_PLATFORM MATCHES "^(x86|Win32)$") + set(MI_ARCH "x86") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|x64|amd64|AMD64)$" OR CMAKE_GENERATOR_PLATFORM STREQUAL "x64") # must be before arm64 + set(MI_ARCH "x64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|armv8.?|ARM64)$" OR CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") + set(MI_ARCH "arm64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm|armv[34567]|ARM)$") + set(MI_ARCH "arm32") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv|riscv32|riscv64)$") + if(CMAKE_SIZEOF_VOID_P==4) + set(MI_ARCH "riscv32") + else() + set(MI_ARCH "riscv64") + endif() +else() + set(MI_ARCH ${CMAKE_SYSTEM_PROCESSOR}) +endif() +message(STATUS "Architecture: ${MI_ARCH}") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_GENERATOR_PLATFORM}, ${CMAKE_GENERATOR})") + +# Check /proc/cpuinfo for an SV39 MMU and limit the virtual address bits. +# (this will skip the aligned hinting in that case. Issue #939, #949) +if (EXISTS /proc/cpuinfo) + file(STRINGS /proc/cpuinfo mi_sv39_mmu REGEX "^mmu[ \t]+:[ \t]+sv39$") + if (mi_sv39_mmu) + MESSAGE( STATUS "Set virtual address bits to 39 (SV39 MMU detected)" ) + list(APPEND mi_defines MI_DEFAULT_VIRTUAL_ADDRESS_BITS=39) + endif() +endif() + # On Haiku use `-DCMAKE_INSTALL_PREFIX` instead, issue #788 # if(CMAKE_SYSTEM_NAME MATCHES "Haiku") # SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) @@ -327,36 +395,51 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM list(APPEND mi_cflags_dynamic -ftls-model=initial-exec) message(STATUS "Use local dynamic TLS for the static build (since MI_LIBC_MUSL=ON)") else() - list(APPEND mi_cflags -ftls-model=initial-exec) + list(APPEND mi_cflags -ftls-model=initial-exec) endif() endif() if(MI_OVERRIDE) list(APPEND mi_cflags -fno-builtin-malloc) endif() + if(MI_OPT_ARCH) + if(MI_ARCH STREQUAL "arm64") + set(MI_OPT_ARCH_FLAGS "-march=armv8.1-a") # fast atomics + endif() + endif() endif() if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) list(APPEND mi_cflags /Zc:__cplusplus) + if(MI_OPT_ARCH) + if(MI_ARCH STREQUAL "arm64") + set(MI_OPT_ARCH_FLAGS "/arch:armv8.1") # fast atomics + endif() + endif() endif() if(MINGW) - add_definitions(-D_WIN32_WINNT=0x600) + add_definitions(-D_WIN32_WINNT=0x601) # issue #976 +endif() + +if(MI_OPT_ARCH_FLAGS) + list(APPEND mi_cflags ${MI_OPT_ARCH_FLAGS}) + message(STATUS "Architecture specific optimization is enabled (with ${MI_OPT_ARCH_FLAGS}) (MI_OPT_ARCH=ON)") endif() # extra needed libraries -# we prefer -l test over `find_library` as sometimes core libraries +# we prefer -l test over `find_library` as sometimes core libraries # like `libatomic` are not on the system path (see issue #898) -function(find_link_library libname outlibname) - check_linker_flag(C "-l${libname}" mi_has_lib${libname}) +function(find_link_library libname outlibname) + check_linker_flag(C "-l${libname}" mi_has_lib${libname}) if (mi_has_lib${libname}) message(VERBOSE "link library: -l${libname}") - set(${outlibname} ${libname} PARENT_SCOPE) + set(${outlibname} ${libname} PARENT_SCOPE) else() find_library(MI_LIBPATH libname) if (MI_LIBPATH) message(VERBOSE "link library ${libname} at ${MI_LIBPATH}") - set(${outlibname} ${MI_LIBPATH} PARENT_SCOPE) + set(${outlibname} ${MI_LIBPATH} PARENT_SCOPE) else() message(VERBOSE "link library not found: ${libname}") set(${outlibname} "" PARENT_SCOPE) @@ -365,19 +448,19 @@ function(find_link_library libname outlibname) endfunction() if(WIN32) - list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) + list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() find_link_library("pthread" MI_LIB_PTHREAD) - if(MI_LIB_PTHREAD) + if(MI_LIB_PTHREAD) list(APPEND mi_libraries "${MI_LIB_PTHREAD}") endif() find_link_library("rt" MI_LIB_RT) - if(MI_LIB_RT) + if(MI_LIB_RT) list(APPEND mi_libraries "${MI_LIB_RT}") endif() find_link_library("atomic" MI_LIB_ATOMIC) - if(MI_LIB_ATOMIC) - list(APPEND mi_libraries "${MI_LIB_ATOMIC}") + if(MI_LIB_ATOMIC) + list(APPEND mi_libraries "${MI_LIB_ATOMIC}") endif() endif() @@ -431,7 +514,7 @@ endif() message(STATUS "") message(STATUS "Library base name: ${mi_basename}") -message(STATUS "Version : ${mi_version}") +message(STATUS "Version : ${mi_version}.${mi_version_patch}") message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") if(MI_USE_CXX) message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") @@ -461,10 +544,18 @@ if(MI_BUILD_SHARED) ) if(WIN32 AND MI_WIN_REDIRECT) # On windows, link and copy the mimalloc redirection dll too. - if(CMAKE_SIZEOF_VOID_P EQUAL 4) + if(CMAKE_GENERATOR_PLATFORM STREQUAL "arm64ec") + set(MIMALLOC_REDIRECT_SUFFIX "-arm64ec") + elseif(MI_ARCH STREQUAL "x64") + set(MIMALLOC_REDIRECT_SUFFIX "") + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64") + message(STATUS "Note: x64 code emulated on Windows for arm64 should use an arm64ec build of 'mimalloc-override.dll'") + message(STATUS " with 'mimalloc-redirect-arm64ec.dll'. See the 'bin\\readme.md' for more information.") + endif() + elseif(MI_ARCH STREQUAL "x86") set(MIMALLOC_REDIRECT_SUFFIX "32") else() - set(MIMALLOC_REDIRECT_SUFFIX "") + set(MIMALLOC_REDIRECT_SUFFIX "-${MI_ARCH}") # -arm64 etc. endif() target_link_libraries(mimalloc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.lib) diff --git a/third-party/mimalloc/LICENSE b/third-party/mimalloc/LICENSE index 670b668a0c..53315ebee5 100644 --- a/third-party/mimalloc/LICENSE +++ b/third-party/mimalloc/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen +Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/third-party/mimalloc/azure-pipelines.yml b/third-party/mimalloc/azure-pipelines.yml index 0247c76fd5..ae4f65ffa1 100644 --- a/third-party/mimalloc/azure-pipelines.yml +++ b/third-party/mimalloc/azure-pipelines.yml @@ -8,14 +8,15 @@ trigger: include: - master - dev - - dev-slice + - dev2 + - dev3 tags: include: - v* jobs: - job: - displayName: Windows + displayName: Windows 2022 pool: vmImage: windows-2022 @@ -43,7 +44,7 @@ jobs: solution: $(BuildType)/libmimalloc.sln configuration: '$(MSBuildConfiguration)' msbuildArguments: -m - - script: ctest --verbose --timeout 120 -C $(MSBuildConfiguration) + - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) workingDirectory: $(BuildType) displayName: CTest #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress @@ -52,7 +53,7 @@ jobs: # artifact: mimalloc-windows-$(BuildType) - job: - displayName: Linux + displayName: Ubuntu 22.04 pool: vmImage: ubuntu-22.04 @@ -112,8 +113,13 @@ jobs: CC: clang CXX: clang++ BuildType: debug-tsan-clang-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON - + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON + Debug Guarded Clang: + CC: clang + CXX: clang + BuildType: debug-guarded-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON + steps: - task: CMake@1 inputs: @@ -121,17 +127,19 @@ jobs: cmakeArgs: .. $(cmakeExtraArgs) - script: make -j$(nproc) -C $(BuildType) displayName: Make - - script: ctest --verbose --timeout 180 + - script: ctest --verbose --timeout 240 workingDirectory: $(BuildType) displayName: CTest + env: + MIMALLOC_GUARDED_SAMPLE_RATE: 1000 # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) - job: - displayName: macOS + displayName: macOS 14 (Sonoma) pool: vmImage: - macOS-latest + macOS-14 strategy: matrix: Debug: @@ -150,48 +158,152 @@ jobs: cmakeArgs: .. $(cmakeExtraArgs) - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) displayName: Make - # - script: MIMALLOC_VERBOSE=1 ./mimalloc-test-api - # workingDirectory: $(BuildType) - # displayName: TestAPI - # - script: MIMALLOC_VERBOSE=1 ./mimalloc-test-stress - # workingDirectory: $(BuildType) - # displayName: TestStress - - script: ctest --verbose --timeout 120 + - script: ctest --verbose --timeout 240 workingDirectory: $(BuildType) displayName: CTest # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-macos-$(BuildType) -# - job: -# displayName: Windows-2017 -# pool: -# vmImage: -# vs2017-win2016 -# strategy: -# matrix: -# Debug: -# BuildType: debug -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# MSBuildConfiguration: Debug -# Release: -# BuildType: release -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Release -# MSBuildConfiguration: Release -# Secure: -# BuildType: secure -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON -# MSBuildConfiguration: Release -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - task: MSBuild@1 -# inputs: -# solution: $(BuildType)/libmimalloc.sln -# configuration: '$(MSBuildConfiguration)' -# - script: | -# cd $(BuildType) -# ctest --verbose --timeout 120 -# displayName: CTest +# ---------------------------------------------------------- +# Other OS versions (just debug mode) +# ---------------------------------------------------------- + +- job: + displayName: Windows 2019 + pool: + vmImage: + windows-2019 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + MSBuildConfiguration: Debug + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + MSBuildConfiguration: Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - task: MSBuild@1 + inputs: + solution: $(BuildType)/libmimalloc.sln + configuration: '$(MSBuildConfiguration)' + msbuildArguments: -m + - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) + workingDirectory: $(BuildType) + displayName: CTest + +- job: + displayName: Ubuntu 24.04 + pool: + vmImage: + ubuntu-24.04 + strategy: + matrix: + Debug: + CC: gcc + CXX: g++ + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++: + CC: gcc + CXX: g++ + BuildType: debug-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++ Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Release Clang: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(nproc) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 240 + workingDirectory: $(BuildType) + displayName: CTest + +- job: + displayName: Ubuntu 20.04 + pool: + vmImage: + ubuntu-20.04 + strategy: + matrix: + Debug: + CC: gcc + CXX: g++ + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++: + CC: gcc + CXX: g++ + BuildType: debug-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++ Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Release Clang: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(nproc) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 240 + workingDirectory: $(BuildType) + displayName: CTest + +- job: + displayName: macOS 15 (Sequoia) + pool: + vmImage: + macOS-15 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 240 + workingDirectory: $(BuildType) + displayName: CTest diff --git a/third-party/mimalloc/bin/mimalloc-redirect-arm64.dll b/third-party/mimalloc/bin/mimalloc-redirect-arm64.dll new file mode 100644 index 0000000000000000000000000000000000000000..455f8394df8e935ddda80feb9acac3fbe2c9615e GIT binary patch literal 55296 zcmeHw3w)Hvb?4^EglBaCJsv4RK!h7;=1M0xZHb_W(7D|?6$#a+M+g;`~RPr z`F3`-0wbID*W2A6V!m(Y%$YO)Ip@ro`CboyrbiVjrN-g!{CTAgAmuMC|6lmO<0!sr z`q!>fUk<)LoZo?$J#1wSSMk((+13&Rf2EGt%BKb-eOnNn2l;Dd}%~Ri|xoULC8e zXWeW`tICyHv+D{~{n(myw(N|W8JbjZ)m-%%I(jcpG{RS zN(mBv1*#S&x{^&sp6i^!R;9j&I=>fJs+g70N|Ft=cq(N0FM-)dElS-a8SnBx z(1$NTI=?;I*^ctAPvbv|%)Yzr;xwdG?fm$fNP7h1Ph|j4q4^wA=NCp(^EH{;@)ioZ z@YjI9)%bIMVWkG=w`jr;+d^Lfq*8zB5aBRPe(|<=9gYPPV5128U*pgDg@rEK&U#b0p zdF8_;C<|^1ezBhCWhIHA7km!%)y1A#Vriy(>YZ9ott=*O7JY#9-r$l{VPG!kS|duW zgyJ2AnocE81)gm0{Hy8+j_c?>@PaQ^)DN$^H{y4|PIF7k)a)(PSTTJns5CwcI=8#1%c z;59(GaeOt8lM})Nb8v3y*tZbqW-fSE)C$UR0)0k5 zjuIc3xby*S3SOrnqwZ7i?U@D#s|V~Cb%5;?=bxZ0w;fmahR_D}&Vww({0HaHe_?v4 z@VnGAj=9LS403Ut4c2AAa3SOo*)YZ;8^&H_vmqqZ))UOcm_Vsc`_?@i({(6n$miEFZQ_`r>)xhrf^h+I%Yq^*Gx$&M(v0pWFC# zZHv&|p=oNk;wE)XYMEM6&-6jH?zseNRm@I&F{3)H}~`43-Uw9zdsByHJ+I zIrQ=c;;F9as{TcD{3Nim@}*{_dB?_q2ikcH{2x&2D_!bie;o{d=XXocci8A_fe%5R z`=N*Tn|5Za*1w<(<~(pvO##0FOaoK>yz$V{^N(mpbO<&iI;3qXAvOiNO8P0vgnaUSYJ8gSY0?rC*kJY;`~GY>u!4++_PJbi zR>$4~Kj_tKCE#}VRZJ{|pc3F=rM=V>38wtLpM z{+v2AN?q#*t!(Eu{&Cv=pbPXvebkjgwJXKB2J;K&#U%9gD>`51DcBb0vue;-pD)Ll zj|OOiEGG`oX(<<497FGE2PF}X^a!%|xpFlp!-}kfgG7no{?BfI)&cQAZ(xX2sITu^` zm={LgY0l4Rk2aKaG*EAbWX9F4X;w4FrWZ({xudeGT8p%#5U z0o@TD0?o~uryuLE9_Qq@(GSKwRwml5JMVG)SUv@`(5s_p=c036FY1UL5G&CC6m2A7 z7gqOam*+PAQCk10@0f$=n=yVm#zDV%oUPB0;cDs*>|)>^>Xf0QoQ!eqWS*I;jO^7P zWgd8oj$n?HW6EammHhjW4_(^7@EAjJquXR0Tw&yXg_3uOn zba`OB_h{f(yeVu$$4xrlwCT^!u6|r>{ZKEl%>yE18QK#6l50&dLnVS353VUb`|sz! zVDWL+6j~n#v_9SfJH$MFg1T{w)k`#I6Y@G z7T}zB3+U)C?7Ss0fn9@8CtW||=mOigiFIw>Z>iQ^%7gPc=nvzpYu*Fz@sxiY<_pAp zf(_!n;wy3ft8!k=*bDj9siz^MmEmPAgMZ$wTFW?ZvYd7_#(pAZO84_CnkO-pw#Ufx zb(iNs%`Hs6oY>1s%lR$UdJE?ui_M#=b&tgYb0Ol>M+2L}&!>RvSljzOx4qMz zjKAOBm%KFg7_+gq@wD5<>t5>&^nvn#@YYovqSd50U`tV(}cf}>Pw+QW9e&Ek_aWS@W;5!ou$dH<&ZMP4$ z5-7mD1^oSSxAh>0_*#1oKD6};_zJ9rbbM8yk5kLS10Kfl6!@M*Tz>-oIk_l2yw~*i zHn+b!vig_sT8FUC$-r~`1@Rn6Em8wwcZ}y3>DUNcV=Mc|Lmra{-}@lxCa`U1zNiJA zoEtjj%8gjP%p}FBv1NyzgRy@Qn9Fjk5qbjMF+De7 z{!BubCGVr<5>wRDJ{d0u^}f%#0=-_9yel=|`;douPI%I%OIxU8`6%z6H|TpAhuGLj z;b->`{J7eQ$$BAg)xYUjje8lzo*E`U1zFSa?w`m!RLm{O?%4_Mmt1?LmharggCpX0 zOUKw}66g?by^rgJ^nQr#k8LlE_I~_<%h}#GJs;m=#vQu+RH&h|K4S`U z`fGCNrH$*-@{db+usO)T3uW(gK{rHZ>{(EL>5JLFVBKz%OWqtu9vs8ZX?ZyB-kbQ8jF=~9|7zi1ec2=96G!P|vt?i4$d2RGf~++ue3d^oIHq5d zvB{Es2=P;f?0w@CH|CQ44u>C(Q+MQ+y%+QI1;^%f)%x7U#t-v)gUm6bk1^-^80@*Y zY~nzM{N5Fbx8P4k=;~5OejKNk=9d4Gb*uY|#ACU}=N@N#aGbg)YkbU{Cwhou&UrFE zE3$0w;1!8ia`oqar$0D`KIYV=OYYB%35nZt_2)sSKR8Z3nAM-19>lZa7rj8@ zOW+G%aWr)YVjiZ5m4=@763J~|;;^=(UGRs)ClSA#xC46x%P<$wr*;PtTTz$C@LdVS z@{%7Y!|{XWxDUt6&9OHzky>GnyK(#}b3BCO&ziPt(9VP5B`0PTh~MU%mb@oCd}@-| z!>-g)#Bob?tQML;JH@fOlb-fQJmH~Ul6n;H#a-!eVOD+#1M)1~XIHTn_Di}M!S z@z2LQ^H`yy$2i7*kgV1Hd2E8C$2i8`gTFtCHsl(Z!q3Kmej8{r3>lXITQeRkXUNQ441{HQtpC-C5Zr1uX|pCdnp?yj*Kmb7!Oj`zJkP; zjI0)O;y~Q7*Ex^`jxIjT58TB#Rc%1o23uz6hmkjI^T@}>aX-Gw*q;-hM$jh(e*syi zO5eD5g0%v#4VddO&nfiR#;Ptqxkd_Jv!MGk@yGr!Hpg|D7jWZ&*>`s1$?bl8Hw0gV z{g#1~)1U{iVfq)$J;U0!z?QZZ7~eKE@ekOSX57K+FNnotER5XMzY4jN$U9o z9@b)<7p@);Tn#RqpM__i@Pseea=T#RU7O%K-^Zp*pPvGslM67Hz-QCnh#q%eoA{u` ziSiklIggyED@Y7I2>k&4T<8a}oal^85}3&K56x@88oD;|vw!t57u%Hm2b2fKNzvr%#dn6r?jTmP3%2zJcRPpX;tY`>0=}FT=P) zR;(M*AAJqyoV|_uQM3@jGY{n zeRSjnur8vH6djtSHi&PS72ZJH19!vkAdj*I`i$>HnO%o-3<5W(1boJZH`q3GJ_uiB z_s}O|PlP^VD`mufu6N{`#5ey%WR&&NkkLc-Cj@Mzof*P!;E!})=3;N{82iaSO8G)v z{$*1hL^*AfYcbk&5_U{G`z!ythrV73zxR2$-=pxVBj+|M?VCVPOe^SL*-wLQFUQ05 zlfJ|H7C2A%3}^g7@7|X}A3|IEWvtV^5bX1$m-s*EuV7mTY$wA$*P$Kc)4vUYHuL|yVDvn(lk3e!*QQSy_f8#-QNQ}<&@Ykk4?;jwoT}d?2DG5uGmTzRw1-|Zew`o`NFFt7C5)@YkEwCAM6jl z^`x=Eoca!|A-|q0M$Q9AaU0*%A^2w6!7o`l*tPW|)()0}&W^jaDbk2d!AF1C+7!lX z24tq4-C*Y*_zLSAEI+v~N89-T$05x7A9xPEh3!jTFZYv~2YZL@40x_CgTLr9Y*O?c z{zCe1>_hL@>ak>-v@h|kNCU7P>J9IW5&n*IY*x*&ow7bF=`sv2E+8&Myn5?Fu-HzdNZC2eF4Fx(yw<-rB+@6Kgu|szpi=C72gDY);9^x zz(!&S`c8X|l651x?k~D^K|`DUCe}^V$!CCt*b;P~JlR&CXPbfHU-~@lN94jDDD4-%Py0p4pVQx=-N2Lr(bW@r zk76$Sh*(zUcHBP&TMw>Z2A=~x+Avh;%{_rUrkuys!K)Kp|AaUSIwJm#{ALs+PJG$>^pPc$9e$sB1l*1Z4dw%(G5rgX z2J?ZXfnL34@4?8nNA{!C#swo~suulV`(}N$S=LvuUHA|7U&e)N8pzEUp&Qsp9_)kV z#k6x{D=O=rELP!E3NYmKamGKg)SFwv@c~ygKP`Dcuu>boIK8qkMzClvQC3vJpmte zqOf2t_dl32Hi2FBsP4NG-!ki9cOH@Zo(qBT1jMqyErfdZ9viJsM&B8e08iN;LBHOB z9-e~yC*jLZY{MLbSm<%=C#C2EopF5**odBF=@fKxAX9J6eZbTg@E1M9buO!K)cHVA z#zp)fuYpS*FoNw2@A@hHFKHk@Fp_nF9Y>Cz?6E*zmj`uP{3!E{f8`pk1npS=MIZAZ z`tT#@ru84LZUl{=vwr3~H$tCqZ4PlsHP#~2D3i;t1nUj*eD(&>Mb5v^m)IBvIz*cf zc!NviekRISbYZ%Hy{0$;wupGupQFxstskJzGjo=dVXWv|X|uo2{@D3!d|~Sz^pSnA z<-hHgZz^p43;J18E*P19voE-&oyOB><+<0a@{3FMaf z;abpp!MTVvR`@aowTvCE41ZkM+CW>S3@Fc*;aRRhX_KS26kQVo|pUy>R`cJyYE}v9UG_yBHwGlu5^6dXFNtD7-=J7v-Xh@Bts@I+5p~uLQkaP{q&-jzy{zv!|o?CxH-(vi@?@yTL_$1$- zFyHzU9J}^!{RtjLoNBQUe}b`4ZH)X^z`kVG6j?F#xA>d5;oc4q8EyRg!Ja!Yx z$;)8ouE(rwls7c(B6PD%daq!@kUT=8qf!mm5Mbq{<^g(r=w#e7Ct7*l* z+#vLGvt+RJh{65#gX)VCU%y3PV`W+rZe5S_`VlgHAT3jH+C^oePtWMbyO3?PzR+&O zmJOb^FT5UQ<%SJTp{;JVa?>t1UA@A`@yeSnEVIymAWi>nd6sh*%Upj8Ou_p( z;SHOA3+=rPng_f|lO6!wEs*PhO_RQe7(x6k=WFWSkBpzPvSIIZ-~-loVSXb&*I&_3 zN&JP_D>XsKnRdN!9mf%Qu+@Q7A$$R@w*@xcxMKRXxCe~3#q%iUfT+6W8H_Ju4SOGR z5HjkzSYsfj<30n|Z(PgR*v`NHVAhxPX|?1F+Q5*Gf0;K(f6lyv>AW)JO`?3r4}zAd z=V5=$(D6C~W3prK=xZfzyy>qS(JtdWd8QC+5VjRE?Ps(L{OKQ=$9CtLc3l}OkuU3e z@+njA1i_~O_e=7b1Fjj=!Rr#g2wCL*ORj^g+$={t&A5*IyVzG8bKSsoy7b4i&pfFI z91w%(dX|sr_ghHAz=8e88hZG7zyI#~(L8&~{u@~?)Zc5Y9MA*M$^C!!S&9#0?|^e2 z_f-q9m%&({WBn4>p_B=-BR-Y&Bm0>-4(VrL`5dCXP6}HQo|g99Msq*>awi%<{iGXH#mKA(tk_Zd6hCxUymhEFd-8@$fwuUYT? z(%baM9AEF=cb+xJ0o=P9c&6^y=}Br8=y&m1vyyWkf4-zL{P~j88@YboCD(vBm)-%M z^FaUI$B*rtsOhB){-Ay$V;S+28Q09@88FPDz>YrjDCZoRM}e)B!6(?V%Iq@uL6+Ul z@#C8G7WC~F$Z&V~^D+;_rmgt)6{-j_&c(IN0V&&5bNG1l|_0_kXLGS+wo2R2E+CC{cH&-$}Ju=@k>A=Z9tt^GnTX$Lv?)xg{O zos*!$b=RZz*>CQbY#u)UgRRFC@@a8Cl`u%lcPVDB%%>qq|!*Vr*v?BccY zQ?Nt17LI)GHEKV2Q{o}Swe%TM7g)&L2!DpT5q=F=;yAbnWcl{$antiNc>I`iCh;PlYYd;hH2o0QQ_ekC z5uT&10{A+)Zi4=P!(LOS&w~BY$11L^Mv=+6J~}WO4Q%zfAmk=L*p=Rw8>N5228CjG&&=haUp3a`fXA>YM~#c1{+| zx_>V5tih}dx|X@0X6%iAVixSp=nwsO_at1if&Q1HuGks(Z((1=gLX~Z;PrB`DddYy z8O*frP-ncdkX*Z*$#{fzj6FNb=%%}j5_7qo1MfwdI)}0pZUCiJ))f6Q=-7y78&=J_h*W6O5e@14)~ZSB}xQ z^+xt}8awUhLB~BE*U#G+E*I{rSeO2yOxrx>1GE!dZ^xyI<(vno>l_!-VodG4F+_Qg z7esy1(T8w6KaP5vi63G+sWU^@K|4o~Hyq%-ZLsZ=&#Od10P6(mCH!hR^vl{_#Iz@V z2X>v|Kl-juJoXP-CuQAXWbA`Hyw_kS_QpBK${LPk<{2US2VNti{JgdxYYgUfr}OAf z=tHcZ<=n{oXUO**!#BIn?8gbjqP8E`pdZ>k>^(({ugpF^V;9B|h$jRijBx;K2hRN` zm#M^4z)kLnKzp8xhZ%3VXAHWGF_CM?+=ms6(I@(5@xkZ=kI4^n6?IhP|9iw%S}y&} zu)@!e`{36Ym(l+DEYnP@&(L3Mb0PYA$Dj3Kj7U!#0@fL}brAB3&+0Bpod0_~;{_U7 zvpV&=ga&$&Fw3gM7+bylLWqm3a_*S+GNi9mSR* zugnYREB%ZgYf%qm?_Xn^r2b`<^BOYk{aaZ+oAyEs7SLW`>$f5;#<`wb&^E?g+Qyhl z+c*~;vTgo%Im-Q*YR0U@Q)l1{a?o&JX5ZF^xd^_K4+3)ho+o(%KnX+D>zPl-1ESgaUPY}+?REF;%O6~ zpGJ8J>d^OQ*Z1g8OnqO^yQe2^Gx{WDW$C$9d@bp_(|Mwo%mZD93|=U)+LZxyD8n~k z8)-k9-uDVYZ9j6>lfi?@p&3_Pi4{i{=(pB@#zHJ6F#kr&qzE^!bek<^`L(&tIlZr zYe83&MXPZbo&O5Z<0g5dGg>BpzHsEHJf{%jP$oVlM)e60fU%Fk|ocLhaCS~{~(7Jx93%;58QRHXqm3^*;_iZTj(>`01 zeLshkVV}xA4?bEwSO2VBT#vE+PiM*1Yvig#KK(7{AhweeKj1+gh)doTzQ;$7AIoSR`qC9&X zef@>b`umcQh5Gv@WHHaq$-RQF3xBP{zFnbzv~Totg#4P`+Q~?H{a@09v?CVEekycnySu30d0L(u z^zSocUtHgmU4KORt0U`U>@<&89sfo;_(XZPCfblb7pehi>tb}k=*=uzzW-?iKhgoy zA6i=Mzx)3666t+?ryh$g!e3%se#U#)x$be^*}(k2GxwDzk)OmqA=l%R@wXXkia?op zuZWFjvG%4Pvt@jqi*@+t_-(`&>2b^@k6j0L!|#U>599sKQ>WpN@*K^QbC7cb@(&=N zv>Ec1UqC*TQ9k?3+vxIT*duIL##r`wWG+Qa&Gz?X$+8q3m&5BlPcg1W9_}AFhP?sooAdd|%=^Ci zyhi<7+V^uj@Ls*CxOOXhjsdU8mj}r?PrpmiS9a)zlpJUCsGNzRp&oE?w z9cw>^{Q>$RTTV3M!Vl$PB=ciy@Xqm@Kgulo-gEmNa`GTAeSY(kPd>S3HTF_fXRA^btlP=D zN)6)tdpwWobG~OC!ItHX*F9m-kXMw>Tg1FW$P1_QKEgcqZGSrNcIK%r(BWYz``+{L zTS~1$-u%^V8VS4rJA}<6`@VC&-@O*qKozQSsxj6WX}~+z=f$IIV)1BQ`@BfJ@%E3*U(?Wl zvKlFiv^U3V)-*RoGjn21v3BOznl)=;ZIRUtLgnX0n?}j1X^+MmDGA5O{`L}W=)`Eh z_o~OgGOoJ4p%QYJHMPZ7w8vvjc)(LVv?hLkwEfF3G zWe+{Hd|A!Cl`G0V{!n>Mb@{!O)#b}pYOzL^FaKnDb#>*v<)(b$(v{VpC|~;N@)b*$ zKY~1;U-=`s*s0Q`_mw@gqI~Hi%a<;{?>>hfY(Ktyg=L%OQ&nBLyt;B_@ zZ_V=Rd&{ep8nxZ3vX#p!YAPSDT3)?!#Rcf@TVB1)>1{T@)g7^hHFt&{UpRl!t)Y9P zb)khzLbsLP_K{nGKCE3mfl+FHqAm2e*6KS$v!CpkZ~h34)f=mn_IzG5YQ$ss*a!VZ znr>cOw5}s=b4TRQTN7Q~vCcAc>tz-A?Zl%E(MX%mu*P*e)GR8G^J}kI*4_0@yhd|X975$5?Yifqe3Kg&Eh*E3)anHo{?u8}ur4{Smpmd?Nkyrz& zJ`roL2m5$?3_f#1DALl>5JR>!7;Ov9n-_|#Zbm8E23Dbl=ExcxYj(=z=ZmNrOM7kY z@Jx~TnvizZA$)(uR*7C{TrDIy>et|_Ej13885FUjjq=5#>pB`Dansm>wee_leyEtr zgKM|)<2PRFqmeaH60M8Xg_^_tih~ z#8tc;zm0=7ZVfqP8S=43C^RcT6Wbz>NAYo;4b60;^-;jx8Efol3_aG-*b-`sJ;|1y z>p|)tWv5mc2O}q5%Arx7Q*bc2v426p8 zV@>T0GB%K|A=b2BJ4#Vy6M7`@(sk-`T-Fk|( zwmBa1d+F-EjyH7w_1F;uu**Fkq&xbaCWM~9mC#@Ho7Cd^uG7!y`tO=PPbdf+FnAvt zBkr4jA&>i5ROz_EJ2gh!_vdjR3+(Ew!8`DSvFYvcJnmz{y~yBQJVxACeQs>yH6~&I zKdMw}@D7g=_tHG>W738ygLnNHaqr6GJ|^5d4cJE9wY8!YBXO-eZ=4$&g1?cstr{J z?^PzP&%*zB?b!U!e`qKE3pVyY49=TP+LMKQPagNnGOE1>=YEqO$ijVcB(Lw61Ng%R z=cGx`WZ`}$ukV)wb~WyV*5}D44Vlz7R-D&&3&s1ImKvPHS$My%jpUh)Kwy=@yFQQm z2y8BAMwh|6Cy)E(gn0z-UW51kJnkc~xttk;2JgcrO=jVrUpHMI^v@cc11C*fV^Z5% ze%)k|cz@H7!MW6=6((J5;`S~1IP(1k+;FQkIJcN|vq@_W?t^*UFAMIy2Iu`I9W-f? z!M!t&`(?rXxWPGPQrk%H+RS~Kv!=Z3F^`XI=6~-Ifxsz^Z;?qYyV7-;xEGu9&3W7{ zMDHnGZ1Akg!u@a-?)9d;Cy)Dk!rNBvw0fL{yDiVBmj{7fgZBZG9?rtQKAP!oE{_!d z{kWd7IKQd!4Vcs-kzY4iB;Mb&$lzRR(uyqHH%IgO{{AAaxz!t-J5AbS(*1QB?{FUX z%K^LUH#iTP^s+RPXF39b;|A|DCJp>B)35YLGtWmb$Y+cRfl`BWl}TH&a6gpCeN4Ds ztkM>PbAJ}zXY%_p7lZCSl^-y8AIieLVs+lXyeD8UQaNexK9hxeQ6Be;Aom`Nf28$# zvPmts&OGiGip!FQ4Bn+C4V(11!M)bR@t39X|He#L8Jz1)+GWySgL`ov_sgOkTMW+q zCOu%%fWf;bkNai8{jk9~Wm4P7@jTuZip!Ix^t@VR(#0kXtkL;r^6RF{g8M3ibBjqk zO==sb|&gOR5;G8mP;I#HHVdH0t*6Q=@dinGBhCs2w zxx%EDTbIGzp6Ao|7lC?%_hys!oAiLeJ)GC~_ZM)(?Xbc5j7f{$(thQv!M!Dq`(?pB z&-6bB0sa1N^9o_Sd)!JPul3g1l0`cG%Ig}T@=r|#mb+a1Q>Q#0)o&^$1C*OnO~Cqm zy|w1f_uR@U-c1~{uOznZ>NgZSd^_Z02Q~6yW7`E(bF%r4<5*2o^a;LeyFT9hgl&fH zIN6#m)`sF5yj8meGOoibiG6C3YeX?cA%BAp-eSG3DY`~i8jxgzbE{mBwL($0#kmnBGzTQO&`sb}SEUc0s}+7333VmNYU6{8QL z9wIjTr~uKf1<7Su0YnyFu+cZSF)8`~M7XI2p{Kl1W>XBdkd zj)auMr!imJaNoC$;t_hP%QdmZ(n16J>?~<^KD~ktTlBe{%H;)^EC&9y-z4~ z8ShMoJn3`Wtitz2R^Y|z6>u6p4sM?N-g^?D>^`VtHoc#hgT6Xi#_z*K+S9c(4&XJS zEW5t1A7-=xtI-6bP9inK3R2qP(b4I*pwGfhSIB7B z_k`NC(!Fr-^JyFq(6EC~PKw;^jDQO`=(hD&pA0Tep5F^YAbf4lY$f7pSlcY2bMo|{ z+fvhxD1y$jHf<@SzWF8r^RmvZ`lS7Ejo|}e(!RiQGwoR&Bdy_JY}zN3cC?*KUgo*q zY+An^tVAE!zSRk*%+JfhPWW5;tg;dOoN_CP&)30{^&rND?ak@qh;}Z;PFzgED62n? z2C$V2l-u#JSY+6r8QkppIcR)5e5=Tz-&t)qux83Xl81#oJ^nsTW>+(C7=>m89y#dz z21%U%{ntfcvQnGzjkrTdH(+ZaJVmKD@e#P>6>-@7OU7->lS! zCM2(_y@mZqddHDAf^WsG_!gd*^s445buG^8k=}r`%bagP`Y`IJkbV?t2pO6F9+?!`KqN#{S(rvyOjE$sHWWK;MJ(9QgD+z; z^)T_P_4<+i0_Xydu%AgUjC2#yL8Og^wS}eCz#Vd=R-iq|UHS>7zKi|`KBd%eL5^Oe zeXO?%zU6`e)R9>@l7(M_>P9OhwnOw9Qe+Fqzd1MoQL{&54)V>J@`V0 z_k(NsOp5nb%Q@cXD|C3jtklE%PSrxx$NM+s9PfjabG)CB<5+;_DS;dG1u6SmiSgDWu03k=x!~Idx9_~+*6!+~2 z9qwnKo)+MK63NGPb~OX-;r<56$NeRekL&J|kLxt32Yp3K`EZ?5zM-Ue z{!#rp_~3aw=wZ^s^I>A2MKNtJD*yX$gn%dCNrmg~%Iq}1391xa;fVa;I3?d>t;N%_ zo4m|gJiF_j&c+7&^}xI5ESz6D2cO}>m$LA!rn~2SVF7SG`~v+xnS27E{K?l~Kx zZFBDV*p(CSinO&w8&~5KZeY;VcK4i)c+;J2_{>vdq-|bftS;W%*1WcT9zF$iXQZuh z{^JYh;0tI?v9-}Q`vp$j8d!xwceTem+S)6d);1e%GcUqz(M+j_n)1EESo;QZj+}V3 zwF9k0*Hp!0kK+q}>!NLbp<7(uiBBE!>v0d^lVbR;AOF95PNc1}>G9_E(fFKDN35)_ z4&U{>d(PTOLtAuCXo1h?t_7pFa@T?kneJNPcMjF>T3~U*$;jatnbW0IF>-5n*6-}y zd3;xLSJCdOJ*)Qg>^b!E;gfgMxXXl}vYIk6Fcz5TXu003#oO$`|%cXnk_qFU(y-Ga< zcFEpU@2ah}TkE%WZauv1__pM>GutY5RP9)`qkhNWi-%r3{9^LO#r@&_ivCso`*$AL zS-iV+ckiD5Jw>-#(VH~07SAK#hWd1j~D71&j^d)4kk zdk*h8z9+ROytiU+)!y2@2lfr_JGAfkJ~fE%8Ka%jzQujvzKXu8zW%NIw}!SCZ!g^* z-o9l=?~eW*2X>Hi@k`{|+fR-~yE=Dw?e5uqX3yC@>gB-8lV7gi+p@QFZ`a<c+l!>6U3w@bIkCM_jPy~=@vlYau? z;DG}t4|?Zj{09skJfasla3L`s@XhYbcH7-8twAT5oq4}`GjHa--<#Rn6RJ^MeG-XF zjU_tTC<`gUJ7bShyjO?-xD13&f$jp(vj;?mc-^6vYCp-W_U-a2Q7b7*c6MPtmsh5q zFFwm<7xMEtrJ`5Va=BWJ8YN9PO2t}KH7c1D6lHE{X2v}Mn?5rv{rroixoKr;GCxTx z=#_O^UlA)Sn!cefmrL-%+}fozmokU^bd_4ks5Go@O|Wo#mU&*aRy7oTO|LQiCd5Ni ztFD>*u4|e|r%_BL>QU^aFrJ2hLN~D20}N4Icc>2a1G{~IM3usf0D|a655nj|x1$## z4_`n${SNDao9?o4S6JO6?Z&IgtL}(+F~*7ca>WXL+nkLz1@yeEOgU5HYFTY;oYb7#tuA$wG;|83)sXc$0g7_}SdNbrLV8Y1WYabuevAfQ)G$g}__JwEx1Oc6e}$Hs$-sb?U{j zQE?FzA}n9~PCk4(CIc2J0531L$k()R9Lsl>ug)Sv7WtYIIMX{40&kh#KlgS!o{rLn z+9q(Pp10{MvHv2L^xFQ!Ncelq^g_7tb?2|R=O52s-P2yyR}OC+G3x=ky1ZVp3$!js z=Q#W3>eg3wEd?#>u1C7n_1UrS#Utgg|LbS^OPuV7n(W&zwEyutzPrYg_19O#esA#I Q50(~TPy6Bz(SH+v04r#&VE_OC literal 0 HcmV?d00001 diff --git a/third-party/mimalloc/bin/mimalloc-redirect-arm64ec.dll b/third-party/mimalloc/bin/mimalloc-redirect-arm64ec.dll new file mode 100644 index 0000000000000000000000000000000000000000..62569b57fe7aa8efbc6f881d6354e41aaa39167a GIT binary patch literal 96768 zcmeHw3wTu3x%S!p6>yiYqk@@}{h-@_ zI^&MZKhp07M<@M$I<7DM=&IjOm2lGUrvNTEI{Ek4%5jrFx(4t?^t1iNk1qfHObK8A zi1S|jC{NB`SiHQ5IO9oBI7f&T-*AhY?pU!(kLeO4J%e0ljT0}S<#Izd7lr_`<;NP? z!p|-t2Jk<;G!#BuQ;B%Fgbzc_NgBjnn=RRA8zO``^Ro?+%gJ7s2w`^Nc$XN)YyB#8 ziCmEIxePH}w>dzMVF<6uy4l+WpjdGjE+$k3s;kg(XPX-LGe9K=KFOB{WG55_s>`d2 zR{Piz!kgm%3*NSuhvO<%_^bR7BM%*G#4bc;NUBYzbR<0>jEmZOV0rmp3O71ju6Hm;%za+^RDAd3$8=kEVN}D&n~F%>A5nL zkui?_{Y1_F47?9!7|r>Bv(FMk|M5gm_5{F`6dyysF){s(iRoXW=%3YR`uziq(7pc_fLBbu~^$cy=enTGv@l&o9kbv*1oPK!P?&p z+1~)|-3e$r(`$65w=p{*y(<;HM|w@qA4zZH(v;-8K+(Ia*Yt9n=_Qfx1n5wowSicA9pKATuv3?K2ciYwaH9Pt4Z?c7{Duj(uN zdr$VS9q`8DYjno9F&5w6l<@f@@oiihm%pb|!k6QW&w;=1#KG5QgmajTU)g33v)O-v z(qVzrA#xe}4kKqVce(I3@cuUN)8D@4Ku6~?>Nl32nX_W(Ayy}%w_ed(-fMbhj zKcynyR}{S)dQH#FNip=2$d|9^jZZ?4Iib;(gUXqMvOD%z^o}z=OCP8+vwja=rzXDv*NQeJ=zQDeKIJqe>=vznye(hMl2g3QsCG$`8`b|Y#CcW?XiC$wYJu5yt)4N#FyRT35{9$@B z|0I{sQ1n*yiC#`Dy(IFzr`GSr1oW;OE*9DIk8%8#-mYUcha2cJI${@w6_e~+f46HqjCqQCKAeEMqt z5Rd*_ivEiyL;v{JNczZ6smlMnqW`^4k^g~MiYfpy$@vV{aBfq68 zf19HJxs##4F#-J~@;4~@f6t8a^Ju?Q*UWuf{|NmbKiH1-mpM+Ye>>zqs$yun(!aF_ zv#)=(TK|7P8TuO&&_~WlM1P5*|KiEeKfX0me&okg<)5PHfA3`IZ%jZxiTuM9{eMW2 z{>tIr9eVA{4RO_BtowV2<((M=nvY}*Z0>h_Wx>kfV#n`<`)Eka{Va6;_a9?jREWIL zZ}ZpMzb^(HZ4ftnM%!@RpVwZF*sx-#*rCTwL@c;10WV4EZ|nvAe$Mo#oecfspN}v9 z(A4C==w#?`>;?Te^8d{pKNeu$IAr%{J?F;>4}$%_NYekiB>jI)(tq~f64M`*r2pz9 z{Wm7=e|Tw<{>mi%-%Qf~-30yG_n7xYP5A6&pE^5X&AQKaZt{<)S-+WJJtOgj&rU3S zs0pL-Ri=zDe0C!7p>}oRZ)wW-!e=KIJ_r78On|QeXD5*|htEzB&*a%Zcr1`Ihile{ z^_f4O{w?%O6MpridGmR|Vm#G;29EfKx(ir%?>q8C0pk@QfTCX?@MMX!5c{5TL_ zzVO+JMK6hbpMVba`P&revHq;J)gR@iAM9a{QTnWde09{Pu1T@yh*-bnkne2x#oj*C zGtW*edZ@7y(WAYP-u6DzGtW+xK9kBvdm+6|eWqugomlje$VYo2y)~)OL;dKmesZs` z^x2N}YghWzHDkp39aZ#h?lZl{NP4J|6484@(YvzG^!#CZs11|J_ne}4L7(a6MAAzl z-=m7&i2=#|JtB6BaQ^AC&hF(s)0ngS_v7G>#23y#5%>_}TWmS447RpEC4AxhBk{?z zduRS$ObK5&|3u((;IBCjKAxpV${fx=^(z19`_5A4a1OHWJ2MA83H)y$X4c1{qvLZd zy>R}CpobWph~5K=URAH@h4YW3C(rK9`|644l`48mQlqy|p9KgJ&OZ_KlE}AE(VL!x zo<7^g8cLew%s--IkL~OiXZKRRh_ie9a|2wzpMef@o~KXr%>1L)&!mSKO?}#OUW~nv zUcWxkGxLwqXG-$XUP!OAf8v-#a$X zHKNyV9%3`;we*RenSUbaAx5k9(>X6<{U#`SoBKr1%s)z>DakiP(W~wgJv0AA&`ToU zU)1{Do`hbGs9AXSU*qrp*gqyo|I8%)UrN$nmZbl_B>j&j>ED^SfB#RD^dCsl|3Q-e zza{9`epAo5m*<%O$2r}*f4`q|9Eil{nE%IRIONy;DdBU>|Bn2*Yq6oe^U8M z6}=_BrstUdlghVH(VL!xo@2hsi^>0#FCrgDuiwu=hxxAjYWo>TtM>xj<(%})33?fy3pd5ZK}`a~}# z|2Jd(+U@y2+Fm9odYk)1FDCydm2ZfmSKTLiG5J5Ke1B2vcY6|g5&6R7n1ALu<)7>0 z;ElxRn1ALu<)6z_!snQO9QpfPO86Y}k0XCsaqz|DACF`HA&(LHfc3B4{s#d68?gT_ zS7QH;rRSJ`C__SeAA%0)z13@aj`=64e7{ok_Vk*bWBy4h-wzbM$CJ==jF+C6{6qO7 z@dhxN;r!Qxn>50idN#&!xkX}Kb=*8rpr1H^T zNbk~Q^uqbaCSOec`DJweiC({ErO)g>(TmAHzl_d5(e%Eh=zVG=_ILek9O9}?zL@-z zRK5yD@0~u;i^)Gp<-0}Edm#zEksCA45p5$M9O)KAxYx|_{Om$4E8tZwJkPvT7dY;qG9B=QRh_lS^C!m>}USoWEYTp!1@4l0w=Z~c)&sD?EY)8{ubyDDuK*(@KcH_iHt*SNMCsHlkPfd%qT)A~IeiwP&0u*zG6mKgKv+ z*V+5&KW{|Q(0dEQ8&x zfAV_2Rt3Gn-}|-6>lOaquf?z^{Mo|W`fNtc{=s6;pYVOi)x4Klc)z!Mu*=&z>5hwr zoLx}2@a(64RMBG$Ufqaitg?rQVB!6uTOe2Q9)(9i?IYQuW&g9t)p-wRwPvHnFT8Hb zud~m(?g0HRdjLLvb@71_S>ofatH1wPw#)wmjtdzh-rm=R1#S@+Xv8Nc52 z_ns?fim&}ih;Pd0&iZ%VLmcgGk1sg@91j4;FQafg$Y<9&rx!g39PN%cKHB*3laGI` zJrw$vN6r9_hYSvp67cbm;gUG`{9E0{1lM#mI->lgzs4K2eld8} z`t=%b)OyC?RqL55UTcKv zcn}wo;i*Z2=N=#M_>$vkNP>seNV#Cwk$BX) zr)t-hA9|DERdyuKuOm0k22Z%q=s%1(NXw;>5$WjDRX+m-~cvZG$(4JE;=>?&2f zzBqAi#~L9{Z1{=%-I35aqPyd4(S2|b-(Gz7c8AXKc3&^V<`Wx!-sDsV?I;!EexG>r zv$CJ;=y=fBQR@~v4xKN$?{kap6B}MM@dD>(8U30I@Y^PxHrz#c4oNhMAkMEgdySff z_Znie8+AO#oY?TQrq9ZpY1(VBso)>`^m_MK=5&w6+{}A2zuUy?z2kzJM&^USw=mZb zQ}vh;hWOZreMc_GMvZrKywNtbGs8U&tHQ@W2fim;>{3b z>ww-4W{B=i=&N%eAaq6kh--`sN*rwG zXW;n*>cGfTPX)V58>y~KyT|9 zTA;yuv4!I|*SqRyrtle5p;LFALU&#mU6LRdQp78TZQU=hJdOHFh z$q#599jWx7?SgXACP<(9{u5|vI{Bbux7QBPpby}Aw}sUG?|OP39pTCNHSJQ{fVRV) zSy%_IyV7|+bU6upNglE0l1I}9g3<=?+2Tp+G1Bfl)Vs3j5jozEx4NXxV&)vdn6a?? zanQxMd$PW}gV(Upu|}4(W6G%UP(K`>`<4FPzlGeoUGqWvj`kVICD-7@h98@L3|sEF zNOTtr7w2@&5>tx^ZxmAxkAdvUu0xQ$^CHuJYV_xP@$n|E#jG}I53sGZ>xeDX zs~7DjXbDp_g*m=_*w@Lzi#Vq40ewdH?zHSl($s?2W@dFcpd!%zg^GX8`;k%fk<3>+M~LntV_=sfOuJl0T`2juz6Ys`8{4Mh2e3AC4A+WtJ^>wR zL$G~mL#A&9rEg)}T*fc*gm&8dz5T;@k8mt?0$+??V=K-rfUdx!8vCHrIBjDe0w3(^ z)v>^r(B2Z@lfJjWjCHR=wi6p(j`1b>6#dCf+hWYI>50Fe#WN+{7R|5e+bWi-d3L)` zG!J@~c|zNK{z{V%*n*6&h#4|Y>)g;lpOn66fq(#;I1?P=;vkWKKdJk~ti9{JPmL^BQwPE_1>; zR!)$gk+)jun^w*^-|UxT@frR=|5!x-h*uAgH@*zXUpW6GcFUMSpC8UVudZpkZsXDf0hYmnpxr zlyE{!}0BUeL}} z4&`+X^{HfP@0NbH@Wh6fV&eM|pQ>B2eljK?#>+LM{n+B^k&Lh$)Din+O>G;~i8@x! zp`J{8kbcfHT-unf?IQ9C@X8q8-&nhvGU)4KW9>)MCd__!f8%EN*NwrXVb*VEyP|2; z3EEy7!DOoYpgR2igS&B_u8Ex1e&uq3s8pr`*d-}Fm+dzIW#(29mizc3+}}j34Xy$YonuVjZG^A5U5EwH zzqRgq9{7;4SziMWeZ2rN1ba5~48Uc6?wsXqHL#9{f%g#h9|sZ7LzBJT_bYkvbHibI zw@1krG}d-tKNg|S{t5Nj+BsRYO26a2W3pM3eF)x~`hy?UhS+=~#|>F!PNX zC+vJ|n02Gn zhq2n)k2k>IWm^F89W{-|taV#QWSZw-KX#e>bcy#&6YpbaLoF(CGDpk6+dK+`>{7Kq5Nup!P)`& zo_sez-s7K{H7U=6at+Zv^p0dd`yd^C{eC#hy2je!c^JFX;d7KY;#;4NsVL#Y|;y@X32UH48g3GJ?=Wi)hRU zUfeHI57O^Zd$R7AHj2-zKXGQWh-XQNMV2ixPDkL&=@-m&!Z*T(51&!PIO1a+44T$? zZzuS)>J<2ctZ!huVgCCCTj7(~7c|qRYG6~6XPgm{e<_PPFOqhTe%Ut0#t%LtUYq=| zez+klUoFmQBIN7n7km?E;Gk#Ojl?(Jh7X@nyE)3|v-9rnAAG=x_lY*%@wszi)LtJk z*IFBVW_*@w6UBQ@ZWQm0{exSbc)!|)51%`)cIKUN2fB^FHm{4d4<=hb)F&m-K_Wb? zSvDaKa{#M1A~9~n}e_1Xp;w@J8z7VXMv3$eC}Le z^25B5>mMH(Py7RenNBh-vf;qz&PAt4ro97$txht1$tDv%cYY~KrrT`%;B)6~iXYVF zh>P{Ya0d@#pVuGuzE0=-t5L%-_oMdfxX%cNo-l%so3XALak2Xl>al}Y<1A&Cw_D<9 z4C7b<9BgyvWCpQEk^OEzKHsQ5@5Sc@>T^z3uyc|6+=$O#QlE`M!Oq(h4@*JsM(@;v zqg*nMGZ%(tdb|JMnbJR+JEtMfPBZf??su>@tvCFGf+4iMVy;g!)^A^Kpg(i&KDVkR zWIehl9RCCtyFUaV$JWw$)dm@Prn4QIq`EvA#fxKdrCELRAB*d89pQTJAZp7F)O5U0tm`^mBN~X?5}&eznrM7xeb^GOKGd}#&RE{G z^F@7I*ZpnzIm70k_>A*CdB#$!^YcKPf8sOFj;!-+(6H{U>igjq4b%W$;NV%FvZwvI z#r?Y@}?Qo^s5zg^6bEAnSaN=h?J8TY?U;=QPi%`5AjpU3YNr z%YDILc?N8&ars$~vuNhKESkSTY@q$o<|&6Oe|*qQf8CoE!>i*z^0(y1Y0}%Jk^TYV%J$>LFg;+2WZ|a46DB#|V-}_hh zKVs|h6C1Y4x>)j$c-(Qz3=Pi3UJ80>mS-wz zpRZ(E_$|m3Lfbe!W)$#@GHpbU!F|-BtPfd_df|7hJ^8so#YXp}p~24aroC~E-G1}B zW4vDTp_c9Z3i~J8j>b^~9Hhtd5NWUY?5>5q;u>`%asnW70Pk7K_&EylD*BwO^P8-l z`4ca2Dd-C~qtKMeVt!&R}P?T4y%?nQsq1;R<#vg8g9Lb+8}mRo<^s zYvqC89mE_9XNSTGyDiZ;<+&<&fPS$Kt?vHjeQfHEae`}nWE%T{?Ekk<;bj~=)oMk`5Xr!$JM`k{3 z5HjKX0698*A7d<#pU}@kU_Io0LKXXI>%138d*``B2L(ULr?$kaEUU}BY z^;15)U)~o&n;ZLG<_2lkqr`eQWgO*Q&vkd>xqUpVuwKgCsB#AS^tp~azq~*|Uhn!Q z(#9D}CSt6LRu@){o$MN2B5xgI` z4`FNvY=`;m63~PFj6>{}`+vst8@Vn@2I%?`u7Q01L)q?#!Tmm7j?b9C@N>eD#$j(HXX~fe~%$P z8OKzf=JQOvhX1(XUtHS;p1ZOQ@rOS5tVv^>^eNzH43zeRwZj}KSB>&h^S*{@3#3Uu z&mv8ZWqd?T;WKX%|IN4tk9?Mic6MUJznqdh(TXIe>t{Ys3fq(R-T~X@9$^E{RiU48 z>Ujt$^BtzG;&1z)&H=9{9G*4x18>i)%&>oBy*jWKp+h1#5jdrf!B)@KJ~>&{16R}T z;gjH{17kzr1NX8vANsin9>ixn-?yLg`;zemTah+BE3;2F%^RyHHtaF&nDGJmK<1z) zj3Cd9X*<_xdjhZWxkKpdC)z%-cBY?;kK3Rxxu&&YyU{lGCB5G0qy6OA?Faaz{lJIh zx%FuK^FBtq$wxE&l0IsWS!ZEgb8xSSHOmwSo6zfT z?I+DOgw4k8Q|$C_#irKouN|<70^TdUNVLklfpY-~&&T<|A`$GEW%^JN?=xVX$YTTT z5oZ7qb%C@OWvi$`kY9$#ezeQF0cZcXhj+lq*X{h|a*fA`^|^?z@;(7*w&Qzcr0wqh z+Qal~$sg_^Fup9(x^al-Gw2(j?;h-uzVT(X?vxe#6gvfyJ=f^&J_; zIMnB@ged*G#NHv+KIytoQRl)oWn3rT99QrN=AhPU&cdF+w(o46scAuv-~lwO+G9Tg zFz&O4&$n_I&x!A5L-Lu2xknf?<-E+*2dQt?b*!@>GoI0Tkk7RC$NM1KaURo+du!d_ zfG(W$YhV9FpY7{FI@&IgcciX%%zY5ghGedm`3H7spq+ZMbd^8Y($!x-F?A(n#aQ>p zX6%yn3vk=>T4rgcy|mM3Ek>`>a;C_=0mVhZABxF)9@jmWEKSRH%U}nzdG4$EOt+cGlYeKz zdaez8(%-TFqRq%NydUc4`E2<{#}>JE7$@zHI%F-vJv-ws^hlY$PMMG^$uIPIK#oDa zlVc{CV>WUO@|_+7`+e2E$8xesKQ>3!I)ZP?{l3nZvAixkm+M2m)E>VMdwdx~Agerw zU_8NR=vn$=Ep#c@7XHb8Iac{8y+g{ zxc0o@Ynp%VRc2uxa_Qta^W@$THLLW&@qk!2-ruI3GB101PJ+*DGwZ_4qYccl>RQ?) zKf|WjFV_t^w8ii4O!RT>RD70wHQ*VxDfj2HU9LN?h3O+Ue3BpFl;fOmgTCwo%p@(~ z;x+w{eDZU$I`ox%>f6kha^H@)e-N?rV1{cP&+7;LIgn9^8Rs zisuA*K52fB2D#S*Jo?$eOi53jSFmQIzrZfI|L`F$ABLQV5OWVcf!v6i@==_tb~0Am z*8M@~(8=~-!>!5e-i%-B`xmq?(3N(I``y}J=}YcRxvnxU^O+vm1|7pEyXPTJ9^n|s z1s%)%pSA<8zfgOAI}YY28G9ig?8VHD_&gSLbi8CtyAiT|KwslHn=SA=F0qQO_;Jp7SE|WXCra`%2<`-~wsOtS25Bt7|{lAY+N!Xq&3{?%rXE$GP>h1q0#h z*bi7a_Jpza48~ZMb7ddakg=T4&U}yZ=zQEiW9=@;NEvkhKiT{5$ykfuZM6D%E@IaY ze#^_E;~tC}C~)B-*n2Bv9n5^Aze_7o!voxBDtX*z0_RObd&R?8@Jv1=-@xA|zRxuIzQNA--i)=Y-C^EF zyqU4~BXfVz3O-}x$kKfdcF~C#YRQ-h8LvER5pBoPFJh~yU++|{+p*9C<>I(`9EW@P2Fgpgkms4^>>b;!lVxyw-7rF-FAzE3UB37MrZN!gd{3@Yx;*bX@77t=MpK zjn(}JQ-1P@^Ki%=jwdSS5XT)CL*L}jMn4lVhWe8I!_m+4bfr7{epd5Femxf@88=GR zg^2l5hj=DK!Y?U!n}SQlRM3;LbOd71L$nQCyYZbf+E$jfE$p*st60COxsZ*yo~C-&uEupy3{Pi-dCI&fDNl^;5ptx!TRd6lKhmd_KD8`UZL+`@2h(TkG~e{k zdxk8s^myj*#5~LKXN1S6!mpe&z;RAIo@Lz7Yipx%M;sYDVdLCWASMlxxy6>Zb)KDP zKEJyV`|DQeSGZ><;|i`hw^QE3Lq28>W1l_lLZ`s}pu`QI{}A+!V$3{a(4cu3Hy(V= zyJygMkqc$qW!|RUeW2o%<_%}%t!L;MhCD}n_PAvWS7x0y5OIEBxGL(oQvmF>0`VOASVRGIh(M~tUdJiI5NSjITw2w)P6i);eL{PDP04~`?;VC zUku-mlJr=QsD1m>8S?pZj-lVlSQ0gcxtTTy9#O-zP1Uryf9F{kVMZ@3L*`Qky4iS3yfOK2E4=T+lIf*xs} z-xa_9xAv2u&$?QEhaP)I()1`kBIv`;vEH(u^!Yr8q-Wz5w?`M_o}=7*j>s1u&jlVA zp10?B;jr)9C-$Caor(P5k2qK0bNt-DYB?|t_AmEaq+LVVQB!ii$MYRYPw~MvId5-x zFJ<9TdiW5~3%yVuI78_^0Gj50D3x=9@blIDejsrt(#JWPKlmZW$!E_gH+>5Gae42X z_aj_58{;00?ce8l1E3o+Wr9EM*(ra#{hB=$P9vLi=6h&(@zQ?%glH2R<3|#=!RU^+fYJ3)lAVr1Na!c)p>S_ne&EO;u>|9rc|UvXiR&I2JJw+0%N#HJ5WkDEScA&E7LHlaGw;(uo>u6XIWHWmbRS~4?z4Pc_aSew z4}UahC}jtJyurf^@53^07hkmKl>uTHcpQgkLt16uJ%tB!U#9MxtonZ9v-;-tSM#n0 zU#y>}#o-;#Eo=U)IIH=a9PW$Z{pxUEv5rl8-ZZmMfBz$sX9swVHL!oi%Y)wx%;X%= z@TW}C_9wR4U-9e^XRW#bOjNbTH29%BA2HV_bUtE>L%;Qm7JU%@GGee;o5lK$`>YO} ziKC5s&#WxH_e70@*sSwj1J@m6z{6n3*7`@*ZVJ!1p$bo9c)ULoZ3k41gq+a9{XS^9 z5bHZ2H_trB>E|dJD-qLZUp3t8$KlELtU+tMG1NsS_$EHY3;AqRJoz&H7>9L8I<6Mu zux=bzhZrHh!;2boywab<1D-#GK0X{IT3P!Z;TnU7gOHQ?*IeU^%{4}z`9#$x3+eNe z*QUD$=&qe~{Ne9uHB$dJebzvq-0SPPqd>#1+tKw!>OFgqGW_gZ| zJqvBNmO4ZqeN6f)&mQQjxTbCytH3v&b@#|;A+lNjF;>hHQ)PRt+5T0u1Gn{EAQ`)G zF3I~6hy~{N`*7CBxJ{a}4QB-Gmt*mn_Z-~ZFW!K?hC2J>_rddRlm+wI=AB|KavOR>nwip8S=MgfS*Gd8RK-k<2{_+ zpv@St2{GHwLxY{Bkz>((@1vea<@r3uBUdgwp&<7bZ13y3N83Sfu5~`*^DfBet;~^( zHJON^S;ztUKCnHG`k^QC>!sgwE%BWM?w#b^+OS`8oYW`rsx@ze-{>`0c4o$7weF1R z4*-|EZ;d(B^LW%EcKs4B_2&WoF8IvF+IC=zMQG=~T;j&M!G>{WX09P^=t1H}JD$Ih zxUpG|!7cd%F5!N0rw86Lh3>jA|8H!CEn)wUkqVt!o zxt;X?WzIbt>y?ifbPi;ddzZ}BdVOSEM!WPit`EmzZ5bQU_HhUKK^yK9;l8g~M{-?f z18*Jmh+16E$uZ}E&YR5LZaxQ%*mCF*5p=n+KZFnP8Sen>S^IvYqL+!ej4QU-hwoUP zZ~24=KEbt=&mkyY8^ITAw=uX!#l4U`BVnKVPB8Jg{U%S0VRCQDw%TwT&rWshqO1+T z$>$P@cfG*OffUg+!3j8Pa*yTJD_OnAlHV z^ZP!;yhd4fsQb*Q7o?qI4T@MFAct$c#+4EJBmaAOFA}_D!Y||*WR$$j9ZvnKHpOqw zjIwNOKKF+XxOlGi&wp0+57t)xK9!;$w$(76V;`E&{=h|s%4Z%i7V&8_KjU}LxVoiN)vl)R`9R4fv1w94Tn@W>0GOgLt2L|N1MRdC|YCg*D+?`CxmD!#s1k5xmpKjr@3xXXQFK~?3IE)J=jzT9KH=_?YzM^qVx-huUh64&T`KUE<%yh!h zVd9vtaqPV?_){AWy$9FnR59(+)QsbZDTyzGjTo6_FbRo@22YSB+))uel$_y#}w-`5##{Q#r>QEj1@f3x|eH*Hr$UpfEty2rAb`d@GAUqtI4?Na|a z9z1ho{?^MO@Fdp&a?$76Zqq%!bKqP{Y2Eh){$_}?{ZYnmM6C~V3ER(7x(;Fg8^XRj z!rs=#u{X3Qvp3@6+a_~Td;89XQT)T!_`7YiH=S!LAV)mA{SAs-%l{CAc;ANgGGiwG z{$$*FjS1#`72j!zNAElK`W$@fym$z{au_@-AC%7{Um@u`ulWu;9@Ebx--p2eVbnwK z0oNg{mHbUp*gWm&%L(v33w%8phWuS-`CHDW&vqXMkM9Bh`x&B@wF1V-dyzR8#6AZ| z+Hwrell1q@Zk2r6p2q_&wLaa4G5$U5>D~vghruiCAAMf)dmh*~!Ed2Q=kLB5z2z6q z+Vk_s=N6@|u-202pPM{)UqpL|`W;K^!eM>ROQcWWk@RpL7oktBM@8>e1Q~fAo`G=~ z%irp@>u|4~pA^^mdZ{;OyIC8LFRpVGZ;U5X;oQ;uIj!?vCqJYO0xxXH^2Ld9e8a$_ zR`Ko&;qUIj4r1>Mk-t~)L!0sDA>YNFH#kLp2gl*}P1wH`H<-6!BZwO@@(z?d#OBPw z;9EW?q1IilIrh-nZ#tso0&d3MzX2a}R6Kobvey&f3GtZwy@Sd9{jU@W1Zk*&cf#d;LToZ zG_Ynu`)#0a%QwievAAhx`t=+5Ln`YI+KxG(Sn`?p??-$-&Rp^^Hih}v<2%yk{I!yQ zhxi8l9!7k5Pu6Q**bQ<9Vlke_1g_XvGPaj^%yo;!b1NQjh>kDz_*aZ#%8JLf{4C=N z`1K$s{RQ&=8E?*W;#JCRuXCdC5XXC%f7rqi>#w*k7q&+=pY#3|bKbV*#oEr;IP+e? zoc09s21|a!&y$eW3mE@nyc7AqS-tCxkTatO1+2k`;PTDJJ--!A1EF7`2&Xsgy=X36K%$t^AF8SskYp~*` z%}()*#Ae0?0X$AIt`_qW<)ddMUT40}174&Zg*#Rz^7Ur_NWLf!Oky*{jmwkfCzKKX zU7I}YU9-=_-pDoL?eVzRyZT)Do$Z!+dnX2uzs2u#g6C1dhCSYKBuuR+gO|%B0RRtYs?P5>@;?OZESxv z7W>H`MUACK<^mCGi@7-L75w{r_{>jJqprV$x+51fGmW!?gjvQJ!2-jD=d0bX61s)^ zRqm+^5H|$=o`}ZdPjOxm=6P`c#;4|v#XRE30sPu1)-^ws-}z372gEdv8+4Pm`zZbf zshoo~c9ZX%AwduJ&py2Qo1{I!t$B^6)0_G0QO88jPo}FP@Y%$(mwV>mEKEFqG{fCI z7CesCHWr6hJDtAFm-L+z~dEZ{n$U!c!cHr=I$@ z<0*==Df0C(TR}CpK01!eIi>cPLV#wC8V!xs3ppV z>Xi-E&x>P2)CFaBTkGSt)+h9+x%M2>h&qM(y1+2UXHJ!VRPW9o5S)p%!r!=;c7n|* zUNs9jmiF@gh#x=QImUSU!-=k^Ke{^O=|9Zu_q6t5`XA5mX*++IIUD`6Fzz~x&wu*; z5ijrR!FmLSEuVeyAU7)+_b%a7O=XPfyEg z)FRZiS*P!9@);v!4v#awYc=Yel*XgJz~7dsUyXYgDUI*&nBzO#sg2k8JF-$6k60%8 z=@^mPc>hSHpL0?h5B^Pko}1cu@Gr;jy(YEsCO`XDSezW8+ z$ZdblFq%Kja5aCDkpKr_G@yVi} zN7S;6NAX@UaRKoF$44!zb5i$nIo_Y@_#r0#{1nGy&nEF3DUR<@{MV*9UdK&52au8< zf*ukx}# zMR_g%=#@TOR_~pZ8b8`UjH`Pye&6cGevacK?5;2NgZfGOx8GU+$^7;HY&?6>%RPq_O-4l|)`Y)z`bQZRo>DPZ14HP{1Qo*Jro-cmo$d@>`9S3SA$7d|uOerf4`&Trj+^Uz%~$X>{NY;pQrGXR>pcn%Rqf`W z^#5rMq&1M%Kw1N74Wu=Y)<9YVX$_<`kk&w218EJUHSph21LZ#vHR=SwWIsFV`Xq&_ z-)rL2uyCJwT^3!LTglOf!A55qefZx7z`
    vk(BRTwB@ng z;}V6q!8HY+i=xghyl5-HyAbasc>C}!!rT7xMqb*xPU^GwF{8!_b-6IxYZQzj;xs8{ zP;5g(JbbZ3Q);PJ`05m_S8$61oty4hv5H9Xc%ir*%a@Jy%zhUaA$-PLLyx*9%)?tl zuev5I#9Kq3x)$kIpEY1)3-QuVg>dZ?!iUfD1=uoMhzV7J>Z(hMN=nh#f+tP0*XXvp z)oV?u>K1K#u`c-j(1g3n?y4$UeTg37_p1@W?*$$cr|#wa>Z`6myBGaeIrZn|=^@fi zd>m60s4gcFH5Y&31k9nmc!j^pf60}&!pjg1_ z<@F`A{FPNVtn*i_xMgLPe_2VOG9P_4Nyf~=7{(@$)_Cr|yJ(HH?=UT@V#PABAj$`T zCpX4cn>=KnZHVdc=LrGxdOprrhZ2`5$L08lHDR2y+9tED8g1Ti8`URxKHCy*qbg;* z+rl+5@lk3xRm9w>vMmpM;eq@KN;M6SL02`nURGHt`#mmEgcv%ZTn#`Bo`Q}E{;RI2 zu7FW^U5K5yuBfb7eo5XW&heqGhO(J$QWmxk6<%+aXmfSczK_=jD>ofKT>HTn>OPqN zjSuGC|3SgMY&mr3@S*>#ap!d;R6Z zhYk%(v2Bp>xN9-~KMJNQ`#9kl)=~TB{q6UD{y5k>Y5r2RiB$O;1@G$cwv8rlE6vYS zP6eOI)*)4!NOjzKu(2~zurWL~?>-D494Nfm{Y9JTs6AZEYt1kH6D}+81r9kb9z0L= z_Z1>uO8oI)U-70o58uC%^y9&Ja-XVpeC<`zj|Y*aWdG@cNC^^uJct}5`%f3d*R3V~ zco5%XlKrO(o-Vy=yz%7H?WapFjaSo&2lcqq1(W$5Ae?)~OL~fzmilOREAb)AUQrcPVN{yWGQYo#>TV0<$jF^S{cy zc9*`2g(v&m!JYjUR+Y?!R`N?Lix*W@6ql|NuNYsbDy%50f^KgquBck)FS#kOrmSMU z5Z80`4S}jJRNWG&s4Olk{ldC6%kW8v;i_lWx{8WGX;ooh8P>8tM&G7B(D zWuAgzJ42GSZxYap$G zv{Amd>8LDF3?obCxcgGkfmBIkOg< zY{m93xMj}5g>z@mQT>yqEnfJ=In!>PvuN6aFQCoBH|GmZ=tSPM>+|O?nltSS3#Ki& z{(9SZpuKLvB8@hTr*Pri1q++AmuXShjAJ zMrOws)nVbQ2$Tf;l@`LK_QO(I7>x+8s-gg_>Ms5#?YGjiVjT|;J<_cBVcEJW4@Ol6 zr2Q4IDlNmIt0#BGx&W=#TK7o3&X_cIg2b;6Xkm1omHy%qOuehPstD*Ss)}*KyWZn3 zFE1%Zvm_X(^jvnC$G@x$y`YWLLQhGVe+539=!6}g#iEH=(yOe(Nh6+H@R*0op5jW6 zo)So9T^;5xEiJ2BYQvI47S~mhzly-BbtV1^MQq~Aia=n3Czr;9-%63M*$WnViUR%> z0S;PKyxdbNPg==Wap|4@l47VbuX=Q~9{+{KdUz#N?XQAOM2{`827`imbdEz^jw-`a zm96wt_)Aw|mPS9iWSv~ZvQlW97kUnYj3g_cK~emMPiTdp=2f%cfjic#_-=odj2~;t zR;()tct(S01%$gMW?aCWx(*>v&6F*p>0P5MFO~0QFi>2TW3p_oTwPpVPQA(~=P_wq z>ah*8ogkYOVaS{SORV(Y8SpF%tS@62Eeb&0)x~Spt?}HkZcVwTviNRNx_h1HTK*p! zKOsm;fioOeE6QM;%lyk%$G7DiA#jIcxxA6Flv3EPmBOrkeZ7XPnuC|h&zPeihb+aT zKCp0w&AQg@V%_qJDpJx5XX7_eSyloSjT~K>GrDpljac3wq53Q@j)NKrnBr2><$O6gJV&EZt-YprmB-!`z#p2|RAO(msTM!&TD!Q)?vJ5ivXzy%|IzlS=}l_^Rfy*Mg;s7h?x~0tOom+m#tX?qu{K-OjA?Kivr76 zldY9y6&_1UyWN}hhAF?fcGLmTIp>40peL0ODgx4y%=>iaN2yvfKbfAQ@6IutYP} zWDp`(l?GOrH!pz^-O)`Zov7BhI-fr&^J#rnQ=hz5aJXtx<#-~01 z+4;Bmoi)E~<;u!HmE|NcdRywRBCFBkwNqexl-|q<5&SC}dSFC+(K1?NBHYogAIA~S zbd;0Z2q^)VnorW2c4cp~q)-DSUd^zcL_H|fx=D_+wVAE|)#Ku5JOMJ4)Z3ebl?BHEr*GQGA!y_blh{ zJtxE2nT|8bROHy7t@NGSN4OC3$ zbicC(KGpi3)3?0pj5XbwOas2D>b{49)e7o)_31iMtKyWxhm@YTDA*9C_a*6i<&^4M zBWqE5-lJf9l-|V;)91WeSEmw&l%Bh^zORXl^X_!Law^qr;To#+>`^c;O7DgLCvESR zvQFU#uhMg2l)g_Pi8Lw)4HPMTSEu#vpqsDDLs1?T%ut1^Je{6px~*cn#NYH z^jxFh76p%~x_!$3P1oAR$ zp$lC#iGRIRV@Sh98W^VZnX8~4>wY>??^9HNXIk%CMJELdlstKgC-y*H)p{nRR4VXIbpu2Zla?rA@)pLi~R*(mhJg`xN|Ntcw3o(L1%i zuT|$`3Th_Wo=xXft&39$y-LqM1#6=8UjBo$-cO~zHLfP5=RHySUXnhS(JDGEuwCi< zohZHUNk2z$TJ)`vc4>XjG~>H&-GJ`tiqitKy=EPgt6+hGThx9jl-@U;7QGiKJ=Z8$r=TX5J_pvCIJIz#(sPS~ z?OOlo^T<=H=qPLi-b32X)$d*9>fakv`u3*lnbV^8G}8YZ4P3tUcb9!k+YxD8Nz9`XZv0G|i+ z0=@v~1AGav2Jkh&2Eexf+W_AM3;`Yn6k{Mipa<|%z;eLjfNg;N&VashAwQrGa46tD zz~O*i{CSm209ybr1@w%A{DAd<(*Xs1e>Pw_;C#Rk;9|hSOCdjC8{i7SyvrazU;|(k zU>D#> z88CY)YMxC_vSzdZFk;6A_?0KL;7KVS>sYk;2VkRPxd@Lj+*z{7yqGax^p z5AajKeSpUSz1KqivoIg<3_#CJ$PZW#I2=%TAwOU_;H7{ez$*X?^C3TA8{llfyz3x8 zU<2S{!0cI&AFu{+1z;Co37~H_5vj zhm7p62+@e~ZGgYUb<0X{WY|Mo)7T28~TSRuNc05Q|_f);1`g3DYy^(fscLn6JN;K@^vBp4A}Gy&;x$o zH-*@Q>tldl1m2=eLd@WNz>A@$cL0IUwd4UI=JX4>TD~R3c0k`FTsQOs-fBR;Z{u%1 zVO|Shf71IW(ubZN6=E!46JU-jlyU4k)T7VHeT?gb_4qF3#X14r1K0()J;RsL{(T`{ z20zCdK^OQ6p1^qMVaawOzKwa}NmJi#fNRn3Z6Y5&S7-sl3K>v{HY$fW(7y4(OrA!1IO@5AGX_NuZ1GZ(+R8!1uQ##Qind zj{9p8f_Z5-xZf%J@q7jFgbaKS2KZq=3gSKm@cIngr;rf8DKecACh_9;B;`2# zF1#Fv-&2(kze6VR;dc+^IQ)*b9Ea~|O8Ht;dlw+};d#OIo9q{9e{G@~a0cKWz+ynp zPld<_tOpzmSky*)Y!jV;_W-u-6{36q;^EJP*aGMSYy)fo>;x1)7a|*eUI;h^umR8q zcnok7=I6d-!X*mU0bYjoJqmWJ_F*q04q{#b;A?=@fNudd0lo|P4&Y%x_rD+>0!{(^ z6tD>JIN%n*e(>LRz%u}k0S*Ps{RQ#>;1a+~0P6rR1>6I81)#g#kg3`(vW7buaUQ>xb^SH)edD)ucWh<*L!=Hk@#$UN+!kv@G z;4h$-7OxCcR^4L$jbf7;P@A~rp8DspFb4%|mTOFtv<5^dn zzkGS160)uImsAGEcqUqSrcaDdW%|Sjo~BQNXxb6l zF>I%2XZ@}%yWG!YKQru^+-GWb*X^#~-GKTE>02xYIMyH<-HpQ<%ePi?81c5~6r@}1Q?>vo2Ab?)lgb!?Y-ddV}sXZG!W zXLo3K7x}J1z6B^~ENooTSky?|?`$RB!fnLazCHJ;yr;ZRH8eFfwKVN%YHJ$S>}k$z z&TF31T)%V6&W4>$J6m?TpU!^T^K{KKbtkKDFd2-%~|Tl|RL4>DT`}4g7yidg@C6 literal 0 HcmV?d00001 diff --git a/third-party/mimalloc/bin/mimalloc-redirect-arm64ec.lib b/third-party/mimalloc/bin/mimalloc-redirect-arm64ec.lib new file mode 100644 index 0000000000000000000000000000000000000000..eb724d7405ec7783206578377aab5158d39c11b8 GIT binary patch literal 3308 zcmc&$&1(}u6n{xtTcb$)Ku;|Z3w}VFG)-a)rEI@wOVg$#DcI9?n_cZfwkg|G10JMQ zdJz8xZ+=jZf+9lcS@GsUMbKN1J&00JtZ!y#HoM7g(gmpl$;|xb&Cc(=c^`9eK++3K zgTZLn-XcBrIiJPYpr@CQ7J>i}0BG3;Xo~@~S444gdNz~Ii=$(?k?iDbK9dzSRg;Rv zQX!%%vZ^bEazxU#R6;3$D9+5K)Ad{6p6ecCM}A^%W=b3#$`65O_NglScBH@E0uXBh zsNkgY1i(xDrB4DpK;3>Gr|N+Z{HW%C(5*(fW;bJAOn4#aYQb)~ScK@n*hp@EdN`BL z4U$8eeL0m#vfEz>UO=n0LmRX}D;%S?sm`260l1{8B6R_&T+&6kw5-%X>atp<5O)z# zR#&CPq9JF3%CZ|KmK9z5S2rYzseQ~}$XNt$coyvkH+w8(WMq}&ME za0?m4U5Hp$Y$WOz3<9wA`iLhRty@+W*KT#?*c8a%A)l~bfY5NaDobUlE6$hB;ovL&zB2&RAqWuLVR9)3 zLnjVS<9S?;-~xnq{MdH^CK_xV4TJVFoDE^3f_*9ra0;%$#^#x6wV;<)OE=5mE7_~z z^yJJGTeo$X(MYDoL`E9sJRF7azm<*}o(n>uq^+Pn8=F4g!GZA|;~Th{e$fxyECK*k zX^A7$0OaHVM-oH^0Wf8fRfOQCZECgGvzHm+!?HVd1tBw z;N)>Ku3MYM<9azxJnj{ru%39_5qZ*KyrBbWxfXY?k=q-OTkIHs*UxY4@8JQ4YdG7n zLF6X6=}6?0$h9^B1Rl8wjuW|EK4CqPI|6$j#@11!dh&kv z+vgEHHufV_Q^;9eaCz|2r-!X-P{YOdPp z{Kv&5*KW5^4z1dz9x$;+(IO^j8$!85CV Y2Ud60^|u-1G~vvOze2oGGj;|40(Lkan*aa+ literal 0 HcmV?d00001 diff --git a/third-party/mimalloc/bin/mimalloc-redirect.dll b/third-party/mimalloc/bin/mimalloc-redirect.dll index a3a3591ff9b07edada16ad83ca1391963a126b2d..7d0ec33b8890d2cd4067b188378a5497b6708151 100644 GIT binary patch literal 53760 zcmeHw3wT`Bb?zQnvSnjg0<54w0tYjWC*a5UK^U+VMj*p+BJjirq7V{~Y>knMEKTNN zglk^}GO2upa@92X>Ncd2v9X20MtRu|O}HbVAX1VD@^X?URUS=`I`pg4CXJIO(f$AZ zICEws8xW@L?I(TT(SEJH*4qDGd+oLS?$9;8QyR^Q!C27SI?RO>`w_2MU+uGVw)~1-1NVi!nZC362Tdb|^O|hkA zWpn(R^_{m}^0UjX`p7YJz3*uJs3p_!V}6AFpG_ZqhYUY*bb$;%c1(pyncmQ{xtVnf zHI1wlqUobE#f^W|bf+mhDHd4g6wSL_M3D0bmD_(Y!U`Gh^f6$(NQjv-D;J7BloBNT z6^SSks*(vNrd3WfCd9W;XVn%V0<0YOiBSpyi|!I;=s!pwL4G(Y#M?~XYY9OgJ_Fs- zRIDR~@^7!lKdKzTrQ3m+{Ij6V(nM2Zsu9JrkQK#MimM)%=PwLVOBI=@%Az2Ss}zjicq$Za1qd94-ulE3olJ16Ppp2m;f6A?8Vn{dH%wZF4o!(LSiI>+*A^Q?G`Ay}Qm|YVWT1SJdsU^;gv)Y1v(`gzVm0zilJSs_Vf&yQkhCwtLHd zwNVIXc(xFqo#WgGGqRsYA!XQI&xW3P@x>SI-jaL1ieyjO_V@V2bH$}7scu75q^gdy z@9euW0*vZGRQ^=4?}1FZcx5I#3-xM{^^DX)>Z3I4dL>{(fuOy+*w09aI+ZdOW~@TR z#)p{*+g&^S6(WTiBu2x^kIwD}F;e3h!=Y5h!6f@7Rt`e*05w;C6XjQtt0mHaWrsf? z(sR&cIe@aIjE&KlXJ(NAC0EnP{ zb+JELWx%rR8N+tZ5QPQ0OS3O4-uYcYTL`Uunb-!MKSj&?^3g;8OMB{i#h_L)*2zwv zTnE)lM(|VOflTUp`@ZTmHVg&TlJA05Tmh$H_sm6q*24^Eykz%$)$XZ$J4o2Mb724_ z6-Y?%N@gOKeF(LF^*n2p{rio^1`STN-7xT4U|yBiGMl{>5vdA>4I))4!-{Q}mOoYN zk3z3=h)dMzg=T%94LT1n982Iod$!u2Zw` zou^`%AS(E?b9L$}rYOSiTn?sr3{Vuw?k`ZJmm0^%cFLBqBw$QY=@I-AowEJwUyx@Yd9NbrDcb|a zT5NAw*K-)qM{9lKXzit%q7K?gqr!mSjB@s2psuH6BP!Ni@$|Z$k-FZJ_sELHzMAgZ zit$M*UM|b){g-yn_l{6Vg8!!H~^A55BU!N+bqYlpJh@alMSw+QO;DN(!|2ELsmO zlr*S_@$Yq2mYR>?d44EiNB9Nk~4ox_tas3}YC%fEDUXz?y-p z>>tUJF-wh`Jl_t!a86Ln>HY(ZR;n$5NnM)Xua?6VD*9F8YQeLAK8IYPKjmb(%(zp# zr&u;2&x2wFBUjCL*&oV;XO@^aze%qZ7U|3Az3d4_q~|yP*wbvaCXrLz*r3jdi1e(L znX$c0N*j`sTE0oh&QPQo6-F|PGie)UY%EF_7 z8DgB{Jq%R;Ph6jMRr`jZXu2pHJk3(=^4qXfdpl^bRQt|vVS#9zg(N!B^#*#qkZ0r_ zcGs8vog_@nvwI$tQD@;PIoE&L-_3$_$yjkQ7=dGm>6MwTQ@-+DKZ2WpFWwo3*IS!2-QNKYEl_=VX1AO=Cn*q=*pgH+e4g{cbmdC;ju1Nz1~rozrf zrRQq>D=ezZ-49bWE%>*bu`oJW%DN&oN5`!6yXiKA&~f?e*8Y3alSqzB($`RcT;07c z1U{ML5tKYuXslibito$uSia+C-Lk9ityM86Xz!spM_zaPeuK`{+_t-jwdN43dEFcj z<~SzPSK2+h{C(&oyDNiziH@7Rp71GZ-9W$E;Pm~9MzJ|pJ)V-|42IL8`~qU>@&_Iv zJHz4(Y@@ne@$>>LSbnkr<7Wv(Cl8U|{wf5XgDiCaJv{ zS=2Vdfo#>%npfbzw5$;pM4cAdjE6w2IkmXBHl=r-NpZjdG-_dFE4c!w6+6#md)KEL&HDgy$A65E!$~cNmd&Pzg@w95AuRE5Ezi zC;9-Mm2;7p^~_WuL!(B3RuL4bGH1>&pf>7bi|T{y9`HbQv!#UZ57@nx>D%EYUvU1g zjH~usDb_=BuU&a7%D==`IX?i&8(8CMekJ4w4%o z@&`VK@?YU};?N#Gdd-T^ggJA+L$-q|U!d6G#fUQo!hQ86TMU1Ej zMEMI<{31CH;v7pOB~Lmf-1Vkj{p(V32N{0j-*PH8xGFYi6&sX_@~~8jy2FUNqr0T_ zD&GVgL*#i*PRXd-xTvWaeO*e9-vox2dz5T)m2A>VHYp{WT~WPV#Aw49 zS{K?Pb?}aH?s+=wo=>P{Ap8c|Tue5-CC3nV=lb4~ z<2t#Ayn9QAbh2M1DWAjxD)|GQ98}4l=_Je&dhPOO5b{sXybJ1j^@5_e@~7x>u1Zg$ zea=ic9rEl1rMvts-*u45-pT>;1Sw0Mi!k!--6fA-NjG1;AAv|+K!@SSkd_N($#GW& zJ$-NG4v0~Xz36fwFeSIbSX+gfju)WHC9KOx8L<5@BspK=TKl<@8nE_d7m>UueLf2J zvhX1k(hm#lID5-}#H`(<$INpsDu7Qp@_)|!A#`+~B)foyv`LYnu6^ILbmwx~o-n^+ zVypfR-C?SdT^7da1JE=hYn zV%rh(rDo$m5Hdt|mOdGj+b23}xHUb3i%CFqiow?$bAq1@E~_NJk})hs(9I|Te0`D0we^h$DgK!oU0(t$XTMZY0XOHM{qDkQLblu-~_*&mRn zB~my?MbhpHi8Qf(580RenpiK#an`3+@@T1YKll8A#&WhVVkd@jPhe{j#_`@C^+9czr9Me8cNA?ARZE&f4r1!yeot42h@Ph zzEVglIF89bj|tRBQ>{_1-tBn@>m(SvT+`IbWi0UUo}b?L$_b#)mFg%%45WUE?=C;> z%na7&cKJC-v3Mx^DBQMF1GZ`p0$xx${{JC+>*5I)b8n*GXN8=?K!J zNIyx-w|gIy&db>jJ6W08qP-IqUzl)_xSth!m4l-|I$F&*gB`~R>vtm0`E7*9gr&`w`OHOCPSct5%*u}{!-89*8Jdx!{f~F=5LmnNg|#>X%31c3-(#36%U1N0osU6Yg!ste z1r6S!#>ZV5v=BY3Z(|i@FQ;YzD~v-Tb7m{E#&L>42QXWvo(orSG^vr8cl6RgTW{80 zr`diUa1ljr5{tR}f)3Ibhn#=VW5@V8p1)-zF@Y*6r2&CDK*NDf*^B5xtS#dzNj2#0 z0+L1+fl3}F2Fs7Fo9TQMJF@a)8A$f89^Tn3>1&RcP(W|xS8OUZ?7Z}|@o_Y2tycO1 zN!4SQIw92V0m``9fIX4Up6gudW}CYL5E^E2I6G_XgXVayA+H1N=QBKkbn~{%5(c(3_GxKo&h>^3({#2;_%9s-TNpIiP3_y zMN(KC(sjCC9f5=KV6%8RRDzfgY$7CfTxi;*ZtR?6--rDtYbNOJKcYH1YKW-pCOTw- zLS` z)bPRmM@%*o2a&Yw4sF>T9?NcauF=xr0ohR5X+I>yfFVvJ3OPI>qJmy)=dmzrfT_G^ z1y?OW#n_|%-@Ht5WZj5kzl~RL1_8XEl(({?NvOV$`TG_cdPYBTx z0%#Kq5I4!B#^aSAh4rus+G&ZMRJcS7Hxs$4f8Q(3=uld8oFRHJ?0jvRYm%H#SN_D9 zb3o4_wf@WPUhUf8(d7+(n56Rpd`lfzgz8WlSo!kG7se~U4Q7)l5`xdAqht|<->g)A^S);Z_A~N4$@xcatE3!hIMUJsTb`Va#&ShEK*e(~%VIe-!R*3S z?$jhlw5LC@(xi>T(+PSl1PuKQX(@i1kVV@IIGdM10%T*doC`hCUao*J7a-3Tsq>NN zU&zPJRJ&{xAq(AZ%vyVs7hl}soxg4Dd@MPkMC`6nA5R0$R}goN{(&uukU+LF;yew# znCEpG`!0NxV%)YpD&_P-PF+fb6?}$FY-bmSt>9u;;-c z;_cj|82HgH-2UMJy`DQ33zLu3EZAe$I{^^E;QUG1Zq|<|lg+**JD;L3^V0E>vGZOU zd6&B@$oU7*0YV8T{*@mS)i>dgFLfc?!=zMqXBbKal)Bf+5_@wPx33BRjA8wN0J<5? zyD!9N2T~xMz!^hU6?+ z3%5}BNFQrIQ@C8f_7PY|rNYiKj2#7OZk*f%F|xQJ4={|Io^|t`4^d!-wF9|gv{x3- zb>8h32laYC*OLlCPk+)$|Nb{h1MTtB#Vaydc?TuXlUt+Dc^fKM0m!Lk=M-8DNQUsUb*aYuQx9ln?;-JE$h1${r{NONUm-g*fJc9|KXka z6}|&WvXhCPG^Yl}usCH>I@evF(=^B=T345r@8bGjk*LMJG!!}=IZ=k`9dM*BH{%jh z*t=b-{fM08!{mjPQz^NCu^!+mVYo;g3|%41fMjV%LMEk^1mmt}n{%_7d;b=eT|?38p&cb-9h|~t72T`SCQwOm z=$7}|C`a+)mqw)k)*F9e^-y{5GFHT$DVPJNOZjME`L4I-=N>Kg@xb#q3{@^)akm`F zu2c8ki)Wh9lmJQes9q#Y0azO9HV^%H?m0uI1nu&cKyOn5VY!lXhM;PY+ZAzQo=8+NHvaeX#NG3t zSr99Qd8#j!p2chzb;E-{ut2+KU1|BTdYs_zo?ios#j}66<$v$>EPyDLH*{(y&MwL_NSn^Zvm0~uz#F}`{7Gk)DZtpA>mT#E?8FHncD8BO z7-j!!6Io*yW?0W9>(L3Usf&`^HHiRiqIr^wq~t#_oJ(j^up_x%Ek-VI4-GlzQ5?sK zQCPM_l~I++q^VM3P*5sGQkPIc!3^qxY5|zso?>5DrUcp*vr)y8ShtQlZCfAw*k3OdsrUNEhI#WDtRiYSk}7sT7xG#o50dq>IV%<_zmy4_BF| zoJ+w59a}8T0E~B5ALk*~6~+F1G-NMG0T<>Vl9_#!S^_rNnc2^=vyI?_^@Y400*eZs zhYDON;TdLmu#Jbi<$4&~W_Fo-x}f&O?gPFw8R=yj^m>)Ebh4G%aw66n6)x^9`gBQa zIS1siD~>c%^3hoLAvo5vu%4fV?b#`}5^NWC;=Cj$M^=)nbPJo;xZ1pW7A?kJ(O|h4 ztWp|wb>Or`H;f18c1tbt`z#Rm(|GIM&>}~>+&v^G!du>#js#5g-o;BEs#fl zYti5G_z38P5q+{0|5!Eb2rO~2F~-VKo;8&~kZavKa(8heJb;bh zcE!E-mdhwQyb~)c%$)c|oN4k8apsMl4$_$Cl|LxgU6NUsGmj<^&Y2LT@F4nwaOB3X zDco@6>gGtN7)H)QLnHlR_<5cL=RC7?h+tUHn^*IFh88<3^aE;VO-{MVIXQL+9H?gL zjt*^kPN`N=wTGQ1RXmP+5r%N+@Z;3Cu@yFQ3PcJ!O@6=T0eYf%&dP^+HA8cE~LJ>Qks z^&?x}Ybei*vraQVrPFwf05_el*7oQmo7Z4 z&iO7@7y&ekO|zVj<0zMKNfU$TRgU>I_d|FmlE!Si))#Vyehvm+-zxn%pLgH{j~>X) z;yld3V}(2~{NO*K7F0CTu0Wm$S*(N+VV&`)$o5Vk^7m1NlcYY$2k$63 zQhJo9_`JF6T!Z6qwp$ua)U`r9@T2Yq+dZF9cf)(ju4QLj{t`O_ydvbK)+6`NLTQnzaQl`S9;~UZsy)+LfMRAeWFQrVNkQN;o_YOxS9&%*@uX70bM(MVR(5O zwOnt9v)`43<-NbqUI&f?E}{H^pMVGNCl4Ux8UM=&+0k|#&YI8=b33~a`OXNYHfk-W zk!EQ58B*toe!kWxZ%5?kveTPkS-$YZe0ZsPUu0rFkgfMMbXCv!Q(Ah6ZGmWMY%n4m z+ao1!)Hf4&>L>NKdw8-tb1BsQEEhy$Tx6>f+2UP1sKCQRuT=2ip)_t(yN%!VEkjz@ zQ>sCLn<+%$Lhd{uyn(*A0Vv4zrnGIY)Ud9SaXue_anfM-%;p+m0!lR6{9K~cAW`b7 zo{t8V?6^U<8l2?NQLUC3fKawy-CEMx8VDj7D`hKi(%>BCqG@b5RW^4_^uj;~g{I&2 z3N)s1gT4K^JJ^J%^E}op%#i*7>nORtlBOS4_H6Dug;_UDQruvxw0Ktp41&(gua5I~ za=2Skd6Ap>V`QQWIE3X00^3s)fpLXAU1--Sr6& zI!|Hd!Pb@OUF1{!;>U=iT=zkvvLiSRbp8UZ!SiEKM@yH_-AWp?%#XE7Hia3_Ui^g z?iE!lxlXMGiz;s;8^A9vcFbZ>yn|0Q-9#UR@enr-D~j8)z7VZOnl|PV?nQ{h0`0B> z1paEfLS~jB6Wp3b8v9q);Z8-}ev)*%?Fv-=M<|Tu$>q5s1_ zoHPP(;ik>pxR4h>r;Tg?FSME6NmJy`S~egei?thr$_k>qO^b)iT%(cRw7{SFD{#UO z==pO`SfmsdgKRnk$iYmt%-9m1FHp~dtVOvpH$DmakM98F*WtmK7SpTK{PIgfnt=$*<<%EQQwf8!%(m?vkG%OEq~DQ3|3!&(!XZhB+Gt z)l5yV&S^RoEXbTOjQJTY0A$0?dDu0mCeY}6$??e8E(CJq$VC+rdws5M)UDg5>fWg9j>Cy4xdh=l)d>9L zthe$?T_xskT4;EB8=O zp|N<)*4As>BZM-)K-#m^Sc~hZ!XmYkgcNTcH3Sl6UDWU0^1Mc#AJtUls-)kd;ysa4 zP_~e=9*d1WLs0>jW8#=sF^wzc6{hjGvC5CRYC<2wTh~})i?L~+*nG5K`L=-Eq&d~+ zP}O1QN`&%cHsH?uYQdF~zE{7R?W|0?siSXX*WekElZ^8ab&{gj(8bovhvdRlg7 zc}H}m1NoKmaF#9=_k=*H~6_-m(w>NnL;D^B~M86RhIz027Si7RVUHso!<7n zR(1|a`fz#>lv9uSwHKo3j2r0z>S#PDB zxYIo^jZGHzU4=}Zn8@U{%KyT|s=5B3RR2lje`X?+)5wQYPxng7XCFn!?Vv@GP}*@aiUy^JmD&z5{nj zyQ(kvJr<0qDrrrw!yK>lA$x)gmiXomDoi9ehM76&%ifs9_C}tcxYDphWX7*>_zeOj z?}^_{D(^3lO?i3imfU#r3Y8}S{5jO~a;eLG#lJyMWZ#L7_^baQFmLC>eQHENlo}A@m0c)vdn320 zQ1c=9l|D1R@Lp6OU${>Gf3N(1ll;G4{>Q5nPBcQS%xHkm$@T~t6qpIq-S)_=tLm@Qc?9SDw{LISMAZiy zapA`9(hl`rO7mXMr7@M}`#I;6pOU!(QDGyGZP0nRTa8y^A}^fFTg5zLGMLMI7xVaT z%*-F5u3|4T-xs2x6X{LKq%4oB_kWuAWQw>?L{T8$hv^S4CruQ2ow6->N2Wh0b@I*j ziP;PAD@vUDl`)*>6Z1CdG{f_K;(UfC--OJ+4*BQ##CZ#JIn&H%y2>Z2Hcb-Zr(~rp zH*!gLzpnQagt+lse4P$Ac|y`NpYlxk0$qMT!ns9auCH2^Z!+ndgzwxUajx%NrT@do zr#us~-UUVC0$)<;^EA?wvk4*fybhshFZE(R(%*GQCLP|7(8#4e z*C8$S)p(MQ?KX4|BYisk$cOcfJVST7(9oHBMt))1bQKGMIq#wdHh|9V}h#;;gu1;Jd zypaDBT))Is3LkS3E^;UmGeoi2+OoB=74O7dl8806Bx0LWOPg9-k-JgmHm2GW8=KnO zV);2OZ7nJ0m^vGqT9S>MS|yd67i&9B*2YvUu~qp%YQn!8=vr=a&bHH>6tE zLF<~fWXmn7L`&P9qC?a*C2owRYFk_F#i4B%iF?+zZEH!iw{1nq zZHC0Hl>_Ou!py-%x+c8R%|Uzu|gMYYX{pO1HE&U1M!qzI5ect2VaTT7I>)BDiAJ zVyoj_T-@A^UTVKPX>C)Z=NfC__ViMH5gKDR#`sNoNjqvJTJW-WdZirQl4NXiI$?4r z)dErG#<}isyPO$x|{!`{U)uZG@Xhi&5Dst z>68VkWK7y$%bjiQaEw-=O*&H8G;if`m1GoV@moDe77Lo8nYHc zXabG9W&&MIRZYX?>6$WUAxGE3+@SYonqTMQppxTl?Nv^fU2J6g7;w_1Oc-Ws=(E!!z+d)j(8|9k!CWd70G(w?vy z)of-Zv2=)8fyNXxHQgkXN0kGLEkJu(7F+1Z?H!12Z#iRm8}?o~|E&BsfP77kJ6A8! zj={01ar0fLA4``CftAf>xd7To6g_F(VQ#reZzWWUc%zJ?jt)t(G1d{s2-Do$w81D8%wsgqKg(ROkTP$xqyR6F09ynt{2Jk;_TIUBDM`ETY3>kH=_KM>kr74 zJe0N2xKuw}V{OK_HPCzTG^S^i8OUzZ?ZtY3BomvNaAL zq|dReKyyo5ii@0kNY~oZc9(M0awzGgHL^D) z8bJ|jOu%O+Ec6D;H@9!y3Zr1H*tH6&&CRjRcTp{lB)64byI1QS)qd4?!~pCjpO114 zeM1vc#W&;;f(Yo);QCs_(Rx+?wbD=1DFi|qZ+nWkN2YO~k|{N4yql+pd*?LnQ^LJN zOS+^2+lgT^~PMcl=0QyV8!QucpPrEZOP{}gc#PUAi$ zYdEg)J~2hyqtm!g3HOs4?~0eE<}doEai0?I0gbmkMchZGai0?IQH^)!6mhS3|J26G zl$8A+ROum&_rMf!{~rX!e`-^{pz+R31~tx^0=!N6bok~Va6;qlOymCM0DL;G-_+|E9iEPCI{)kSmPbhVYC4M>3!3y064|s4vllS4*LplpWZi35!hF$)~|6M)ZwrW<9eSuIK7U1 z6$GbNJgRX%sl(E5sd=_v<32i#`_zDam0Fg@IjF<14kH@(zG>WF1;MEmH)x!jb!fN@ zeklJOvcr^r(8r&CPyf9^1iCezdkn6cU&V*>aUam-rPH{-LBv({;~LMA0^By8!p;ebXBN z_Y76{Xq*RiI9!PPw7s7Jy{Q*D8s}3wtoV-dE2FoouyY#sse^llngNY-SchjIoh~|y z2sCKC<2vjv!2jg5y`M$2bF2n6&LcWJS%7;*!M-(Db2|1$5UBdD^5X#=8eSD2m|i~` zEZ&STr17>3aF0*p{${}K@{DS{;{~{fr*U^NdNU%O8t=XW-210-e>32AdG>3(2X&Y+ z`0Mq1pAOv$(~+}+zzL1BWAHwq=F^O}_sBHvX9ed%CRG~efDS`C3~IbH)3_Hxc2?36 zjdQaO4bzGbPOl>k7H>w_sqx;U!{a&(Yutm=`=&Poc5y=Ed{T#}bZCT4p932t&Td%x zJ>^d<9fou`eI9vsaW3Q*(Kt8juv>?n+RsF${Y)WbXC*zLaURy8VVaqqR}Bu`?_!y5O#Y24ow+^3oT*C3$YAFf{^jCY?KE#zy( zRkq9`oqFZ=MoHx+bp@7txwuJBc_OCXR7?ga*GWad`h2~%;?DQn$|T-R++toyY~-pp z6nprl1(CZL)po9{So+1M7loA3H=PPE@`#88eWThX;7QM?gv^^QZwJMl_l zms;vIp_rmi#edV)hEJ!9PmK7(_t1*?| zo&vQSQ>@5(1&VC+R-Ck0*E4#yZ`qQJrNG8x7#=+fiqQsX9#U<#$4z?7kon#u-WRVb z<@8bYz51)KcnCrJQlAtQ_e8w!+eYyO zJ5}Y1*kEa(0exYXlw6p5S~}g^>TTf?2N!4A9^F@-a!+eyd(|M;1dwC8K7{y4vbXorOY6ifE&D`m0Z9;~E2P`>Fq zPnnxnfSu%T=nKjw@bi=#Olr6rOu?w2Jsu07lrxl@{xDeN zjX&MFh4m+)aq)0R#U%D!Aj1P|zWx(=7}#_D@6zNqH4lf=&`iK%5;|8PiKqXjjz9&z zO@*rhUq9@k{;o&@X^f?#Kl{M_yDfs zh({6rB`yp6`w^bPRf@Q{5c*s!L;|Nc@ z8#2is;i_xV4#dL<598`Ud=TO8F!V>f58=;nMG-%V@Yx#l7vdubL)U>8@i4-pxVjO~ zAp9w=X3``4mm7c~;|R~Y5jrAnA^Z@o0mP#SKY{B6;{6CqY|tWJfzbC}=*T$2jym)s z;++UTjIZlv5RW4K-p#-P@lk|#tq1mq#}SU<3L}0J;k@?&AH*vV`ZmC>7)N+LzRxy- zcm&~f^}vU5gb7>%Ck&kk58^6C9N$3@-^Epd_$b1;x1#S5$NS>NRk$p~@y>qHh%10N z-e)d8jw^^b-t{b=#uY*w@7ostiYtsb-bE}H;j$6O`*Fn`xFU$-ovq?eaWx>0_lSxY zaYYfwyDf$9{lFIS3WPPd;)vrnqM{jB2jX#rAIH^+IDWG$Uc}XnIPSBFq6YLQ=@I@3 zu04q3cco$gS3lzTO|^KLX~gk6UorOs$U_{@IEr`SI*vG={}QdZGKk|DJn;~&VZ`x# zlK2X)5ybKAhWG`p6Nro3g;iarSZ5y$;;u?ANK;<#rkK7y+XaokrCM{!w*Wo&c5H1mUk|Kaw0voB5k7;3FIC}NN!5#Pt-oPOXc50zg^$>^;%lYV zi|&af7hU`AvN>xSlgZfDP56Wx7_=p;7o`(z*Cg?orLB$0C0koIC)$(kTT)B#DW+>0 zlUtW=TfPWiKx=E+5=)vdeJW{SWm#)diF7ixu5C-Z=C!GInUSUh>9-T%` zBKDCqq{NyciI#2n!rz^-q+94MUfY3B9rEjOb@-$gzU#;T)r%UF>)N)p-xW(NveGRz zn>XY8q1B7FG`1#Vi>zfXpEb))FJ;ZLJe}4oa~p^1YnB<@keJvV6LaE1_>nvMsPm|G zsNu2bV|yMOdHlrV6^HF3kt5wlGEWRYL2H0J#NZ3<4e#yPJNWSNhfh8n*dN^AyubgE zfk#FksXAyKY&h6AuxDUop!871A^TA0W8IHs9y|58I2<}0Khkkz@QCxo$tS?0Pl#Fs z(Y?)kPdq&Ou=DV#ha>wN_DA=}_YXfZ^2mutoJYb3?Sqkn(SyeaG6TZ{CkBFtLWc$( z8+^<<95}q^NdFPh4?hGR75!EHR)4U+dvD*~J$ncCihZT~D)v?Fv-WlD@7&+LziNs z;c)Ze%#q8v-dvxd}p719{0|j4fpi0OBiE?2;<|74IrkEpMOvOHH_lR z-uv`r>~E+4{oDap)!)ypZVbn?NVIi(G}x@w2U}WN2tHXt0Qqja~krU`->^Gq9OSzbrn1@_!4awdZKAe9>sto7njW2|mTv>_M|GPYp`GTISlAv7XbK7N?T5<$k+Tmxl$ zP}q(@6<9ug#!^cng0Px^L|d5~@y{(pNcu@$SkQHnF`_M8xBt#T7mY=!2^>3MC^`tS z<>M!L(NI$>2zQe_V9MoPI)UyM3)>%M@)i1wTsoJrkvFE}>#H~5YZ?rHWZt#-diFYe z4ZaIskJ9DO=HY7uT{!Y0>|mh16#`+R(D zCEms8j*(81w;6pg^4pv6^%Rw^q$*2D=&#UyBX1G!5PD?fnFaW|frRdQ558W39dVt1 zPrvlNRUWs#x7t%$wYSn!R)tGV?>JMgcUO9J9a&mc7rykaYL8#UyoV%(-9VQ<)^-mmp_QCsouYx7R1aNJ>G4v2rN_n7vOwY?n(Qk% zRp==>RpoK5em-7oUMze*U6}r>6)Ci>?A+T%E#&R%gt;jA%k-BEJ!AptnKmGO`i+I2 zSeXJ_(=Qp+yPhXyyq)g!tLIjCrW%1_(MUTPP3}5Hud0fkV|!BZTlHOQ|NB808rWhV zfK{b`MbPaWDZgK(=eXv+SQ&-ewRk^Aul zZRe4;ulmB<@MR*E@`1sVo+Yn7NLN&W$M^z- zW(h!1AbrTDMjNl1>g*~$xC8Ysq^>0`d{lQtzY(d}4Zyw~r5yWfNPeS1%Nru`No)0` zzfC^Gi%w|sIDRlG1$ZFliizl8TVRp}Z>W9LeWDrm6n6aE2axJ6 zdi`yJp?DNmWW~AKVX;c|lDmWtqBVp}sYvqPMXl{Klmyz=S~_RRbQ3dsGbxCY%Ko2;W4m z)hj8ksoZLBIvo=eGs z#57)zo~Ibl6pg1PNPeO1N$!u&qL>{Yp#mI49ZA$hA64@-4H$oXk*F0~1%+0kcBl-0 zXKWC8KvP>iDEiT_v=TySVuaxIXG}TQm~!|e%HcjEei`*lA{Tu=U87wZR$QOos%XXFJ~w;~xJQy=M< zIRTcKwjwohfG)Ys@MLCx35vfMm{qP^6Wlqm*)*0ILJ5-(F0=H~pfKljfaXUcdxJH{$@ zUM*AiJX|bNDGj32@SqTm6s^f75h(0!;`+SgRMF*cGOTbC1!-TN#ri-Li}i(QE*9%A z{t{yd&DVJI^AFF7_LtNw zqP$&e|CH7#?U<^HMqZ{IAPB#{kF%;JA^B52v6M@=%LyJIT30mkDehWu)iE|dUP2~@ z^$#5XNPH^ND#7$prDwTD+6G8lTvdeewTx1Jm;|qCAy!->r6q2nNy7&fmz}rr|A57d zl(RUoiYnl1_45MYlNlL4E@Qcb>O4T)kdd)$=N(d`8y~I|DZ{7lC*u#Ccl+MrgP$Yz zpov&h2C<6smiTB!Vl1&z@7nF5(;U4cgfbs6qRoDV{n zO;R`^q!J~24v{n^VG^lz5~N6C9u|CE4#C4V(O4wrrT&(FbSbAL_K zgQ#fN83Q8#rWZ%V;GoA4yGr)FinE$+6$6DXz56qsA?n!f;>)S75B$_zLR%IU#LQR9 zun~|6d*QnZq)8o0=yOh@p8T_p#z_i=XMv87O=%@EE;`{1QtS| zjshA$i%1S1K#PzdyIv#~Kt8(U1wK_3Iq12hD^?XHd;Sl~?fOPY{JM<3D@`FQm5`NI zeXqBuR!zL&bVjvuQ#C&4LA7$B+8R?TKguVK)?}$Rw{W6rMI`{{b?kk_{$jgW_yBFD18=z#khKB^ym8 z8%Z`gyHO|!bCTnuNmbQ^sxXz!Sq>o>{i{jV_ej>uyFKkhjt|XhNuKDg_9XT0eV%q) z8Q&r@Ir~;&no3q-0;LkY>oa0{?k@gy7=*itu)C;)ssR0dBIzzFm6yHZ@)~*BCobp7 z%YJdWLS7Dt%Qf;cMVHWY5HWrDL3=%X5MS_ei7_JUp7TT-V^0yH#t5J{}j(XcTj#^*eRYs~&}ebZPC$KyWI^5!L!7Ba*_fD{1%y&xy_intf(;}GEif!`BjugMC znH_nOae8{)r)fL|ypYxedyBLPG2J|(yvw;|s#E3laiqtkleypM}XCieA zE#k12$EtoEPA+gRiJdr|OZL2mmdcT)!6VNpYN```KGf9nnJlbKlm*US-xpZgMujYqLp5@b2j-bKU@vX zz~^fOHRq?dO=Uhl_2Gz|oCtWS-jf^xmco!v!R z`r4AOpeftNd5>vCg#nfFk}xkXB0=RPUs-?Y>gONgx7fTD16U)|^zcTAlo&L|{+(Ou z2Wcci$x<$bcPMUTU@*#SD4(ShKQYtO(?)iv1FiFfD9az%FZ7pFbmC`%zO52XvffMD zb*2yNOPcfs5Chwiuk7}ukc7?ADd3HO%3M6sEJ$Q8!K7&}rzbziR3pao%jq0P*t;78 zSK7Or_E9G{d8`s5!sI^-dKy%GVNMA9SG;Q8xq(FVBaYgLLZjhR)xFQt1F=+9>D@R+K-EN3L*o{W2v8&HjMK}oo|ACl#B;wD9)a1e7YSVxeS7a*_-lSc-tN>o z&NP~iWxQl%SUhZ8BQF@l#2~&zP})Au5AP z)zKbVy_8M^M&Ao}M)&t1nY$Q1JcY&Q7@h$l4q)KHKmqoEn?&E2!pOmaO;@EWVjDVM zqW5-`vp5_trINE){G&YC<%cF&<4mJ`O(*IbM3QTGqg&i>(Zu~05YWjHDAV8Jf^dB$ zU3=4?;-}4x7q>Ijd|pnb??wADzHlFs!#_Ufj&93r^AMSJkyJ9@4Jx(GZzKywUq&c~ z@%oLDrE0`_%^B)X%o)ljE!|JVUc*?TranvzUNQB3ZeLDi%4vQ=E$4=l#lP=5!`J^% zvX3`gZYH*)YBH)`%A}4Forjd4Ql^`iH&KX08kn>rjAdfQF#@3&P8LL7DLeKEH9*KMOa=m(zIorf_*G&CjI0UH-=T^z`2p zmpx~>O*=uKMru9$A-ZyW&PVoIM_SRyG;++{#bftkeeC@wp|MU3+~V}{IXBoCUIV7o z{Q>UDC{;r#ZuefY+VYa!elAndH3q=~;R@ zPZ9y~a+`=y3SR`O`D{*&STZHe>ES`WTi#N^VNJW_ufYr{_A^5?Sd+#Jn6^k;qnDm{qdPUw5la~Jl)dAOZp7$x9!*6D5rt?pjYBcc2aV6- zK~eCnMAKHXowT*`p(*mv8VNd^p@$16f1F79Gt0x0s0wHDg|4+}2uUngi+R4P;sqV4 z0(w{wk5gLbo{>W!ANeA_6bE`$iNYx`ODs|aLR43wt0Pr}CKS?LqC!qACeQSibd935 z2!X$5iJMh{MW3|zCT?M&CmUVf ztGvQ@WgzF7G?wB;5R<+%{Wm1#tcpk*p15_;`Ksp%2&7SmXAY$=!l(FTA_glPC` z)%HvtjbA=B)JV5NV>Q+!hc%sXTrw!11k}j?aazr2!?6QD_K0H#Pvc!aR3R=pQq%N~ z;W6@boT-kRgK;Mooq|a|s55OOB9=O58#I>jYfq~2#1{mZ?8-!oI+EB%NFX|raP$S~ z>t#vGPwM!px+*Zo4T;Py-Dhn3E*wBd-h;Tj`15H7uwN-BR$i-L??}37dZAmtR0n00 zv$mprNC68f`&u!v&N(91QU2bP>{ZfV~l-W;&L(} zpbq$rAED*x>nr%$pPme*yuXH7abn&gWMMr)D~NoDqtGF%ep;_;*w6D*K%nbK@@c-g zfGodp{5yOhKSxdE1Gqb7`~sQq2AISo)_~Scr-+yXAkD(w3`*`X&IhtA{Tj(cBC-o3 z%0#{a@enb~ZU6Pr}K0wDMf!D{!AAY!mCt)D%CEcIGpu#Vt$jGRP8-C62V)#&x z@3NA8U*HAY42;WYJo+;xhg#@hXyI?h$Ccj}LU4Fw=5VL)N6vgj!~MPgWjeRX%skxT zLm=H`+Iyu-(PAY8rNEt5IXK)9Iin*-A%WC%?@WsJb{KA{qG%7Twz!I&cs@Z15Gqcn z$*ts?ggB2CHJO^@p)lk?^EcsjMi!!as+YO}8;OsobQz0(mUB^@8w>d-!!E00Z$?p#eaJz#?oD{ z0N7Eq5hVe@v-0U8-eA6T)!oKPJm!ULejZq~6POCiERx+mEgPLr0%WD4=ysqp0-lu$ z1X!m4L6PCK#b$8AGQs%E^v&`_ID`Mvyft(fMH4D98x0R88vSnpG`j9v+>c(=T}~e7 z`#YGOsEY|2B$8%0s_m8 z|MyMa5tB?7G+$t?&lgOvwaluB7Wuf%Msv_fk^)!j9mRAUok25-xy;vy2_s$;H|6l+g;```nYr{Maeh!! z+x>+P&7;Mv9|v{;;qf|ly8M{bXf*TfpFjPNoUo+(2fQloC*fpE_MC|D7VO>T=7b!>QWaUW8eh>c-}sem_*Q`_qaI(-oGvSBl!Y z{BXcHd)K?B(@H6;wS*0sWp*xMQ^T>5ZG`~pAK6xDLY0bEi++$w%Jy{yFiwge+klpD ze+e!BA}PA-MY7)uw#WP5bZhpZ*>MKo$~z?b-)teY>x{;4h~DA;4eMII-sMBnF9w`? z?AcGeeu5J))1dAyex?f0Eg}KkbH0W&N%R>@kwK3RA@gZ7^UW8{TkG7x2v-B7X&mC| zb#hK#y5t33>TL2s^o$;Va885m;b&Un3A){XhGOG4nDN9O>_Jk<2Zin-g+7m}h>}=H ztnEQP{&dbF)8RG>rF~duP^!_`&)HKrKf^9dh@DMhKZpD;xs1;u!tKBwL|AGU@I`dZ z|BSg<_wVHcZigY3 zzCflo(hdni6%1LrZ}u!URO#C&%atf4ZZHY2?-QGAy4K_Xrcn%KHQpMf zp&%QM24yyo7A+S+%l}E+r#^?P_mtv96HwC6?_iS65ApO7@W_~@v`(I00TD`{d*1063LqAhewpP zx9Ho*f~q)W*-C2>9}48sCUcK3jC`@Qp$U=JHE|!UX)Kw1L4y_s==|e{@Lr;cWaQl> z@x;h(M0})DHNv;@6_i)5pu8l}YpbRp{e7CIy{v7utvIW~`^^f)kmmLxe~INyKdbs6 zGrlKXNah>RXUWgG{>-^1wx1NDg*ZoBCkF}Li0D`|$l~}(2Ol+*(RJk@tmOb%%ZHv5 z0|MiRm&_V8-a}a!Kfd&11Q&$S@a!V$`%#2F!Ui%w%(pZn&ZZ?Pzw9pFj(wvs4K!r3 z3a4u+(I{@&6o*L4`$3U5Z@QoTEdDG|@#2Z=0X~XgLF9KHdR8<^qXua#E`E15f*+S- z5kE)2m70LlJL53uFtaW|=4P3x8-8NO^SeA_(0BpI5(JNy-ih2bCT}>yyTarxl)Sf4 z4THv?nY@1RmYKYZGrZTDyn|om^%n4qLE}RvFF|*UXi4JzkTc4q(|pzg7BNh=`eq)&up6s&l;gVM zIp|XJWEo4`FVbBBH+7!h*#8XhLJHGBMVW;T=ZMr=qU$bxk&yUXPRSuimh$&fh&0v` zcoaYoW0oPIw_1d1BRAtj>cYULpz-qK3**^Y!a%-^nBR%9?~b$ZluVtHsXYf%F8Uft zAIL#Jas0?iu9r;L=3uf|L9Y6YnW9m{i0zHCw_lQenc|Kd2gx@2Zb`o;-rl_UeqAy>lY_~EZ@%*tVLpE;LU^(ehs)wzpg~VsifHB>$<-&)3cl=grb&Ji4E=dcvPpV5a;k@0IqZF^7~cfd5oQ1w^luHJG#WJQX0 zShoHVnW8HGooSyave%C#J5@*L5u$#6rtdReL?dtuRovdqz-5svk{Nki&KHHF%P?f> z#{v~Ahz!1VN+HkI9%8cNJ3Tk2;IE{+`&@{1(}Rksm0zP{{cJD6?Tc@c@@o3jM6L`B z=fHntGX5M|UW0X{P0LJunf7JtYX_+zPo?LeZ#WMtS0abLpPP*ST=iX<$KKCD-)r)) z=F|6=la0?D_9txTN%}#4w(U>_`Pyf(^y!n~!`qjmQhR^?Ja_ftd6%60SijnZyq-r0 zM@oF1t3E%@BPu_C4t@SDkC3zL11C+p5?W5v>1_?Ql&@NK>Mg(V4Iz2r2}%yAMF0LM zcQ4uKC-~P~?X?p9{+HX%kGL!K8{fkVB`gz$-}p2#(l?}WIIwowdYlNw%eeKK4u5>k z8#rq_>y*Or&UP}GC~=H;X}>Fddm7tQ>fXD}+U0WlC-|K^2t@t=mk#wN*1tp6pK~2& zF}bW{W_(+f;zvXQ9NWJ{dk&}+Fuo2AI3Ojvzl2QKBc72d#rCxs$r^ePg{X=_CH5m? zLwUQPbAt{~+7QKuC54yR_@i~Pc|5rg2NR&EAP#^*nAuZxY(Kgqwp#J`{9-(Tk6U*X?R@$YB&_p|supXwZ1 z2LT5G2LT5G2LT5G2LT5G2LT5G2LT5G2LT5G2LT6xTnOL@X|j*{Z=3=UZAYlwg%B1T zstk95*_8;#QFdIG2@2!b^D(m9Qk11%*8#I0G3-?wwZueE`Ev@`9M@XrnpMDNxi&D@RQp|ixr*XQ}x@gRlbsA=TrIi*iLt~i(O6aQ}R?yI7}r!pPou?u6V1&o9d(fu*Rex z%AJpcz{NrU&v($P3D{Bm>V$4z(VRsgiR*m`y$IKr;{SmVu0i-Pf)^o${8tfvj4*~U zb1#@6p#c9-SjZ;Kol6>;nvh$|XVhT4HCo%y+7imn3Acpfl%wX`+JnR%g> zJXy8zP_&tJa6S$Kf0zjPlA>J`M%OAW9u3xqR%v<5L(yn!bdvH1g3*?6%XUSP8@~J@za0)nNYs*?sc0st*=}s%a^UL-gMWx)gN8A zdG$v8+>e>`>+Z-Uj`>#KR#CNi-Re6wuHJYX9!fXqA@-Jyo0Zs1eqhu3jhoh2ufJzq zZPmt#%G!;aD%WjdEN{Jmit1bS+VvX(8#h&No`mkUjhk+@wAL=~w$N5hTc$1dEnmS~ zx-}7QYFMSs$9rA*=YeobL+b-EtquE&aBB-n9!xBefe2K8awloIUfc?fXc&LOO5QU= zw>TE6PefJjg#5(~p{s#r)1f+)Pbr4^1_^EM9vErc zg5f5xJ`j#KLVPqH#>}xp3q~SMVPx|PLosdfVlB9}6{V;f7->zd!3Ly-IA)b+swf2W z`eN~5OG7Z)poxv1hX2D=ETnbYvL(yv-^VG%R}T7G{A+Hlg=;RUl!%c&(a`opQ!pxp zFWnXmg_h(HYLWlDnpFQs+Y$%H(ntttr7um~=%4U|!8p1O_So8xXbQppwbp1uD7tC_ zT}ZGdS{9qElr^6Znnhq-q;19CdX?r&&WF#l)~jUYieab(JS;61YzyJfjCZuQG>|=_ z50c^LM6>pZL~}%og&%}yj6dxr`erP_*A&FJJrvg-h=$`KErQiVI7Y)&+twP@f(;GP zP%Ng!uyhG&-e6ptAK|7^+f3qwHiDB67|4&HA>l>Tga?yIJ{V?`mA3?~dT?tc0KKSY zwKlne7X72G7zA5`_4nl;Gjkck3ag$k#A*p;%e4g6ELYdOje^+P-r7(yf)OO8K#sUJ z<3xQlPBkb)$ut94wl<-Q=FN}IogbS=gN85csQXMWk>#DKcOubH8_?C{B9i(>_#rvo zh24?k6{xS$>hTZO=ri~a)eoZcd8fcXm|D!!s7s+tgep>wkuJUS!^#IJ`fzguKF3rZ z|CWs!xk(UnhDZLXwqk=CWMA`RQe8Dp0Kzie2>#65GQ70lHms0Q`wUI27DG6C6#6O`#2?MWLCVgENy~Y1c z%ojpT{?={VVxf4gm~OfLB54u?@oIiceUd&J;v-({4Ta-U?cfYq^=9)}T9HexaDx&a zlCR{NQmhoY>yfV1vi+0|tZmpzC-kV7@Rcp4Jos193Q54rZF!teiq4W_$rGKfN@YFV z5JaBx&x(TABYigI*{)5jO#d%iAoa7<%$p_nl_e3alpy8^v-z_9F08_37OX7nXlaDvEO++pGL+Noo46 zLOs@U)gMZStnp{-pGjy;Jj_v%Wxk?+ZTuEk&sMHr&-A}pPqxWr^=lrQ33%k7Qx&%V zoVC-L*M1oj?|FUh7X0VM;}EkxiTA)1agW^P;64>I8kBgSo+9qO4(?Ndof#7Eu_@v{ zpKT#&*^A!?t{QLPr#_2VQw{MEL`yJe;gu7qjT{A`8dmY@TgnOgJyM2nd zvyV9InW+HZBk}H^BJM+uy-x*pHYo8Po+9o($KI!e`EfVoNSwzM-oMP=pE~=di-UWq#95PZxr|da0)OqE?0&d7TKw@z4M?2pWSo@o zsKh?XpqgnY7_PHE9mlxgyXb z@$Qpx%7*{&$Fu#-xyp6u&Vs;S3S3KNTqa}1>P`s6}K8f>y4ey?Nob{u^=wif!67SPC+}Xz*+%E>+O6I7-`wQ7PryP4% z5M7kGRN}4KaQ8d*eo^pNLVXf%zl;Mi?w7dNZOI-7iqDDP83K(G=cJ5#WE_#WYYy)3 z4B$N6eG=yZ84t>MRN~(2;GPG$ca}XYaUPSgDeYdzznEBDghc5V1-@Px>oWFB-0Pfu z(?vj?u||n=yNo+!tSWQPffW)LHr_9B?w9ePjGgny3yZT|+OWiVOvYt@CHzkNy)t(E zj2*HIa$PQQu92}4IwWyd*G_z45lBkB_se)d#-$$@_TKB*`-KHuN*k6qv%i+>8X4C~ z+*1zj7X^1m=>HA^^!`=z^|E-Qx6;Cr&9jJ3zNVK`nU|7*%H^9~Rk^sdl&km05&@Sl zUIA^&=db3KI-uNqFRy&Pv63s_NG}9ay$Sx8Lbd#5#FCxlTe8J_ds{*e(0@BLMq3}K zwa{6z1zk9X;@ZYgFal_{Fof zt(SOzzF^Aelfg^jt6t;H4N4T}fA6!jF36i;XX@SCN*;+a2Tw+>>K>j$_?tB9-OZNw zMx%IwoyvD@^J)}W3L4PcvqU|%+`M$Mx5d9iSl-tz^;d(~Ql5o3+QL;)e-MFsR2P`- zRU}zB@xiU6PV(MuIoK5?$Iq5W71{IE`=2vd3kliyE2)xS^%QT6kRY))2|2%b2RP5O zU8`!@rg+n4ybN83(=c(c=Gp)61ZB73_5OBxGcO1Irced_cWE%5;S%kDtO;dye$#5? zs6xL zFJC)R7JF_0xBHjNXbYB(9k#0x)@eSw~qIVk2L-(kiCr1i!}bfj=hc0hcvsJu|){|NaKIL z*mi^gr14*CtOp^5H2#N*y?`)?H2$N9y@@b{H2x=v%|keiH2y<|eFR|`Y5cDV+lOEv zjsMzU-$fWj8vp;mX4ODfr175w>`nx>2lR+PiQq=M7x4f>Dbjea75gDV8Pa&y5_|VY z!G|=Sk7PjvFVg*p{{z8?G~Nfq$`Hzt#=EfCEeL+3@qQ+@6G2BBzfEJO5CTZ!cRTD1 zLJiXR?G#&tP=_>rSHiX+G$M^>?%4r^2-5i71N%Ng5@|dW&Sq@^&Pd~Ve0C>7C(`&$ z1lxzugEXEOXJ19wk2Ib!WN#w$BF*kaT?l*gkjC$)*{cYrkshvPY*ig%qyvbbMldKJ@!LV@i?s7`5O5H&@Yj3m5eW(M!6*or9{0{x{d0zTw6JQU)r;^5O>Yj zw}j&rEwS+Ccr@Iyo&8e*@-o-et#`GA>suQ_vY5S9gz~Js>edW7d^)RYh~6HGS2i{2 z!Ip-m5PQjWM?4U1jfd*vcZZ|#M6hW?sJS(|gRzxVKWOiW-yMp^@Ql$NiRP_HG1f{_ z*0m&>Ls8u7uWAi8gc>%qHYA!tF*cVpxHA!o?pW8-7LK;IG>2N^cL$^4;8xUjKXKlM zr)<`>#G^ZIYmHVkG(e+Ou_Xu>xdZ={q5RdLh;%D2sg8 z#U9EG(QJ0-%0ytz&B^8_txb%%wF{Ol@h#9oEg0o+0BaZARejsy@&#Hfj^P_@YHbOv zU9ck*TX6GD#WU6fW3f>4)}|dA1hmA~E=WXMR>kTYL(Re1;^uIDv^CbcExx$EwRu%A z*1V)`*#fOO*b?3rim4GUYJ(_ETN95aV)6AY+ghcxd1sNfVjkxMXDn2oh=$`kTA(d8<*Zqnzm_#ivvgXs)NCBs*DO`I;lk_%C*dIAAixpm!QKpsz{#4E zJtvXqWenSRHrPAV>ptK=s2^-UIPhrd(XmH;hszH~4);CQ|CsTZ*5~c3>)U^%_sHoZ zrN_#S1&(zd?>RnroSkr=@SjMYY(F_bviczx(P-~jukS$P!N|eBgTs#+k7|c%4%ZzX zdhGOLdS9TgrmwLtb!70!(2?OI<;VQT^kX&0`i~DBPaPjR?mgi zSmUwCvE;G#W2cV~A0Iu=o^U@AI8k$=@kH`O`-#DmLnnt%j-DJl$%Yui(F5}z&<_k8 zOdT9Nc>194Q28PMp}?WO!~KT`4i6sI9``=(d%XN{|Kt1ndizR`mL1iO`i^!U>p8an zSl=;{T=oP>?K#0EV)H6=cK0?Oh#Y7?FnrKBIC^mGAbYgtP~D-%Ly<$tLqms8A09qz z93DNaKOT6z?(tOLVBb*R>AvB<@}vHv`q7%B{l^B5rH&0A8#?BF!uLe^6aFXkCwfox poftU5MscPdy|ZU0%dqTV(PC(Bo1Dd=Joz_^15?saGKI(E2rZK&ktjU1g!01|H^egFUf delta 100 zcmdlbwo7b-1PjaM`aORp%dqTVQDo?=o}9&@Joz_^15;o1W?NQWMwavsuD>Sdu}wh8 lc5?6lWhdLRN=^1)7GiwV{%mH*$0`0RTALBn$um diff --git a/third-party/mimalloc/bin/mimalloc-redirect32.dll b/third-party/mimalloc/bin/mimalloc-redirect32.dll index 522723e5017b71b458afb4152bd6c7eec55e9e62..cc661036ce98ad08a4d8a037639988548e285107 100644 GIT binary patch literal 35328 zcmeHw3v?6bnfBP0K>>l3)MVYnO@;&lDVIpLi7)a6h()1AAURf=O_PAYGLrGFx^P>& z*cG%E!drHmY|||1(w5Wk@1LHfTNX${`EAM?l0)~HEZfqSwuU;*IC2}et2Pcb@&CN< zHzUc$IN5ABi}$R~8EfXdyx;S^_wSn-+5Xr*DMOMZ9gdL^Nji>i`Yh$Y6aTr0>X~=H zG*dcq%MWHBPh0(i*;QU&KyLCkZt}b9<&Ex!hQ^@0!6W-a4YIF6u2}o9T;EvjSv+IL zv^gs2O^@8&cW=RGk7q_57FZCUZ z)M1mP)oGbhX>Ii;t*%!}pFKTo=EtQHl)NfRd+)%v35SEPO?;j%NmE31;#*ppCYXfs zbZKcOccOmOfxpGqQqUp3tge)#`BW{ZNxkTL-N&3J%>m<{=iqKxM%W5VE}v@4|%PL~orzo)Jd4QcKYc);U+;ktIqN06O<@CZ&3+B!y2KYNsj1kW~1z?u>1iFhpN};``@PKIs4Timyqg z?$(v~D}_txLJEH5MTl0~d|oo1{5)0c?hNVjI%Ao~=^~W^`iBP$QJbC-7$WwesLK>_ z#iKS;SV@cEZ%bSX-UV<`*U+By_B5P@yV4?j9X^#7aSg?OfbJSDrl$|L=%vt|;TD4w zy5)>*h=9kY;$#G$sBI|r7*|a_P9S6UaEnRWVgr#q`gZ#Wc))b^QJaAZ-i^8pTmY0R zCgV~2i1DcFUF)ksgYl>h=k|9Obj5VBf6D9BfT%no{;I5t-4eA8hdVJT;h#t=_J@%RL=;|}P#pE(D>yW3cfiCHi|vuMPiftVJYg1@FMbBrf# z5NdD+rfza7w}X+;ky^825NG|w^XM$19B~a&_bnB`koVf#Aw#=s1ieSXL$~ZS7*EQweIPc?c+xczK2;EL4S=Tu4y6kuKve4%vFT$+(cZNC6&luMs#N|?wBU&>unk6R z@u)p+wPy$Bq|7}VZL=zsfJQCOB8bN>Xo_)5)IP*21=Wk6f3wIn6ev|m4()0G1g1br zz;jvP8j9KmGi)d)Hns1e+Nf(V1G;H@x1Cizb~wR2=m7Ya#Ntnn-OrTw*!As9L69Z& zV)|UEnvxJY>x15SQjb|Lh9(+Mri1NspdFAhGzB%*^P#L%rRDhGLt)ehw+Tx@{F!gPjl!KTB;A8r{QV^CBLX$>W&4EjzKfc!TQ z0U*A1n?Znu%njC_;2m^z^wvt{^P=8(vJ*lzSi3@*7(R9fW`a4gjp$h_FlLPUvT9kN z{|MR%hO9|2qyki}c0+JJbw8T!Q08dV2cve_D8{=@E9tZjf?;-*I!FlXjvXSpM`3k} zV2ttPAkimkKh#RjF8FP<`^nMv7#<*3;ZWQ{!y@2@T@MEZdv^^p?O_rWDC4?P`N_Lv z)8y}HV!^xNj~u`#DxY8)#C&jL7#`)s7>`2yW&0<0dXMIm*}4T!rY8oCDIk#$Owu0# z?;}j&t{XbY{;v`aa-Y-KNA^ef+9a_T-w8&&Z&wKehhdYeG?S}zD0>8(z+G~iPGyI7 z-DCY7+$lT*^nu)IHaP@1vY9A*7fKJ$W0JOp^E#hQnH1}@KFJ3#0o@GNS3^^PQm7Fu z1z`f0Y#Gp}At`i0lT-A^`TL7@w zMo6psXEGLQzbG^pRLDWy7&)br0=b!5Uo*o}GVJj^w<3r{ZK^A$;dj3R+W*L{5k;!+X-0w2y~tj|r*sjQ3CTiq<#5E_ z7j^YjDwKLa^8M2jgET=B%(iVb8etBF{(>@$P93tIsNEEG4o6(QVirxIySx(V(zUl$ z?HH6o(;*T)i{zSFBL$j7swoKHtxQ8Dr9BF*0}!_y5dAfHXV<&%^a`9#uT#lkU|F${ z=;$Lbq$t&)M}Q&RrHcRwX0v=aVjJj24j673kXmNqY*Exc@b-@>a>GDf@4`^0Pz+1&`Ysw;jqX)#~4)!G2-+vF@lWh zSwTv30oUiJlz)NkMVx3Q6}z%|5SJbD&>+IV!<>bufRm_UBG)PXZ8xBzX_LsOKK zSO)IHc}B>fY{0qkXeX3-2D$5zS=1Cngd;g+kQgxVrbGvh|7_gTl-J_OJrNkvj7KwR zyhMEWntYO}R#Lj|B~bcN-6?;7zZOl>l&^^nkh-2q?aID~x( zk~9j+N+P-+9w5PB+IZkuTom0eCZRkumP_HQdCiNjFD%;m(d!D|G)HDpgc~*b<23qR zsiR-eM!z6NzmOb#LK!s^r6YeJdsdm#Lj;I9Jw!6x(!#jVKbItz&AVnX zs7`=VDV-xqdw-w8Czccqf1F5$TaGiRBO3i9g8q?q*ER1Y>bzD&k2AIxsL2`IVfyN@ z9d>lsUvMgIh!>og9*DXQBSO4@2ywva9j2bMxTooiZCB0)3196C`DwV#M{9O$s^aobb*NI+XZ<(XxvJhc z3*7l5xI+sGr}7g6_19@GF?yGZ`e3-_pcF&~@45yLa2NVqL!TMm;~=g4(>okDyV>)m z#qLAf^w3AV*&9*!RkaSmEjGhjO+{Z2MaH8uI5NBvGq)nEP(Yv1f?wh}6R>cBZM771Sq`_i_R>?>Quu2{Z z7lYGSxF9c%y}`>EJ?+aB5s_vhqUJlX#azGaA#p0(vhoptY5XI43(Z}h1irR;G z(8MkdL)Y*lVkcuF_+vblQh@8R^-kr}^J!MRzTyRo@wg6??drv*%5bzIO_!2opnq*z zZ5j!JwF}Bc+L;&-^=$(y!7Gdsv7LZ8Blga2+vzm#3)R3~kBwj`BXT3ki$YU#JNx%h zW7j~XGFz*|{s+=Estckn+>#{)=h0e*leFU?LsC^baYAXCw}%KH{CBodc~_k&9qii*{px)`9<9eIi&tNo&J_pg9>kL`3) z;3RJeyK9=fbI@Rnyq&OH)*ehEv5yJy8&4Xr_k>m6JRpfK*ClM5u+Jt) zYHEvT(&id$lI1fTn^glmHCSdrNZ4r@3U|&&rSVt@nRwiI%sHh}>E>)%XFP_zj~07v zruQJp!#?Cx&b>*4Qi7Jxn6Q&H6kU#WXjpV1g~t*NW!T|p6lEE-)dsU0!2ZOT#5aGfX|t zIQ~*A=8wt#LDXWh9rlY*$9SdkF-}PIUOiTs9lLvg*P;9$(N}e69prOp?2f0Yx)O-u z=V>uC}p5fqEtd4+6o)BiJ#7XwQl*2FMVOj}~Wd05nJb zq|0{mOu78W7|0@eR5`)7RzjI9HEaO4(7X5n5K24)s& z0}rN3h~=}8+(!r8nDTG4Ndo#xB`8t>bS38!cPL-h#KDt*ts{JHmk|aaxAQDF*p!U@ z9?=mSCXk1^$wI+)koR!l3eYUup~S^1x5zeV4DX}vw3{&y)$MU8KNeL?L5YGT(8)^W z`~A!X7~48XBf4^qQ#qW{5Ze-ih$6;g28UC5UQ|GV=`Krjd5%NbmD0A?c#Lkh^zvw% zQz|aOoyVs3uf<&oO}5M*y%IvJ)CfGgmTrDHlorxR+Wcg$aB!YNy0grnGMW1u5YuS$ zLRAZOqn?CB*2{+8Ad_mtLn95L+#`Yz1TCEj{B2k0Mm9tgfuY4yg{Rq*M~2^8?ozAC}eqvL6Z9NRv}4@Sx#&WIu)PDqTwcB%Lc1!(0>PSt{g+E=Uq?1 z`}sb0s?gidP!r<+>Yi;*5(#RJm-5mJra9wXz0 zmFw9}DcP)Af@6Z;s1niLcB1sSDwWvIA}&~EJJZ_VKvc&tD0sz-;wQ)Dv) z;A7$lJB|BQIUHxY>8ACHExokPh;ba|FT*{@Q@kZ^CX zo2V#QFo=ErF}jnOKO_oJ%fuFHf_QFctbnQHw_VXPpdpBgvoQ+UP+06dBQF}W5t6E; zEKj+^*2fdj93f|yEt&n_0TJ&~N`K;-4_7)|y-*DW>j>dAwYIy_HfrxJO(S z(+3d`fgub0Fv0W<@YUe`)36eUvJtBZ)k>SMTOkp=F`I!9AoDjz-ThG|A+$N2)@hX1 zDcj&cD1f|zf|>}d&=B6HlY%+Og`ni=SY7g+w_{)l3YMo^&o*cFPobD(H)yOug}w;s zSH~}*0r1r4uc)Jh`_ruFjUBXy0WM=Zn(yctcfCvn_CalP`*=CTM^;4J72$Vw?)^yQhv*A|jsg_dOwFvhbyLypt6LRBG zXIl75dDoRG(@!Nz(or(~6itP@XTV$Z{{zSg;DdCw_wV?c1|riie(6j2D_SQ8!%ZjL zctwy@1o)|4mv7@OoUSWqioz@t4{ezKDyLQFd!sIt>e4ZYL=+Y6W^XHxfRV*lYpsshYmvFxA)m~h)*s%}X zUNr-&A8o|(h{KYGp)75$wrbtZ_qJDM4)wR;n(=p${GlBfinb1L52x!*+&McngY-2O zKXRxp1sKkTFPFEy(u!J5Dce`ig2tm3^Bz-s{t8uM>V2coZ9bntED?$>A}R?DmS)v< zs3B*C7K?X)R-Dl&)Kb+^WTBi`)>8Wv^?MYiNklLp{QDbPa zgkK$5;KGeFbwuM)ecL;c-96?PcNlpF39)SGh-*=-BVdY^eL^p4T!_rayT67 zY0m`f{*Qq~6v;85hr47EE|8iR{gljCeMkrWaep3rJ4Pl0fFPrz-Uz%Plp}3PkJX^X z)Gc?^PNl`0*Fo_-_!A~_#%6-yz&w%Dnj$N6#HtzQOt+$m7<>>%bc93%+icK`I&&yB zjIPWkM7V2RsXQ-;(Y&$uxTInf-k*wpc=KAViboW^InX)g3WCH z46=uCKeT6<7)5(W2>koGjUlOe!`L|JM;?;Yy5Weyq}G#=Q6Ez;@kjt&z7t!5L9vp+ zz?I4;1;wNZpvkm;IQg)W$^F;@h9kOZ;y!<7^G5{-;^;qI>1`+R{r(puzR|9qGt&3P zFEjCrSB7o*MDHU;PX_HNsoSm(r^s6`4pEcPW{y~vseDo_YKuL8nvXqoL$ zKYXv`Jxn7)>Pkam#0}lH9#A}w5nsay|BCYq81dh5{lieV?MGN*T}0WBQTG#!{!^S^ z!stKK*Ulu)^WLE&Wn{ABnGU5sF>=i9Jwd}_QKh`i`b-mD-HpK2ZTmHb`3*?Fh4VoS zi-&S-zXSPy$K>KqV66$kbkH+ozy48jaSI<-et-dTbXBy0Be@lynFJd$0T-aUi)$W%^ovN>gA4bjO%* z#D-=201Sav2jaehcen=Xx8-M4pM_{eu*gxUDWcCFKYRiy{TRD(N#7(~z^{zMBmr(LzCqRHey@t3F)KPh1V)YN2Ly z_W=%O&5JZz?vMupR$8+bZUPGemxu+~{of@lD#%^m!Ub;c(@i<7u@R{w?dVPE4tSnL4t7GTiFl(Hr463`k6?sgEn%_bG$nToU=lJ900CPRy%!N_Wp ztN~tx6_KD%k{V?9MfekgdhujB_P^Pra{5jziBTeUbX}<4~E+x6)7nITtV5 z!C3A2P_~lULau5ItfS_1LyI>4br^STtS7dL_LtNwRmg=NWoB_T&&Kj9sc(z4?%-ZX zM>|ll9(6~Eh^dnhs-VS_@IW?Y7|rxNoUWN}E6oJx%{CJ(O5ahHmcua?yS7L>XrAP% z*l_e#mw`)uuFf%pNQQPf5?DaSXoxuV=W^iXo|X^M-_ohKO?FqL{@YG@_~bSPg@Z=*hlvb4~2>fl6% z6DM9FkuhQINm~f?;4pR|Af7=G9oF{GpzxGKM3(D2pbJ78DwuzHi-t1^(I@#u`6A6m zofP@&T4XiU8418wMJLRFSf0z-8y5Xw3Fbk2h{(l|0CjK*LL1a-KlgQmVMO-dNX}IC z5YKax>2XSAQDu8SHGsO7kswYZ5hn3xd54K*wLPrv$7$gdyHj+an`uHBp`sj9V8rY# zcs#?`ABTst+Vm{lPILtJ$P6mvVOJ&tgSJDU1_8`0T#_c?fnsnX5f~8Z#q(7vGU7{p z?EA4zLa#Nmzr6x*#sSl}RAkNrUP z4P&)l%umIyF#&3#h9>ra#SjTcxIGe{kp^S%$XZrgr8npWJ3sS7_7cfr<;6Fc!7#|! zG46k>cQ3X4?Z0q2_6R>I4`#9t(;&$6JCz?@Fp6 zPHqt$9m;dOTS*jRTf9|NO^@TBVXLEjX^wY2b@WmVrHrLo$LeIqQg0K}H93^OOzGH4 z9owmfpz5@a3z8jaj-|Y@n(y7mBfy&eR*(>P__1YXVk7e&K)v#SNcg%lDIjA5ew+Gw~0>AvEQ1T5M%( z1=G$>rHv#XwEvm)T8BnVy?Q+AR7liwlhif;Mnf2P(R~sg;FzGM<^X?1#gNk#Q}EMT zCavTPLsB=(g9Y3`AAErOe_HfMstHTB+Oq=K)yP^vafZ^$3}oZVk#l4VB)-_6$IRrm zF6)E0aQV8LJc8u^(`-T{-PN37bn-vU#tJ#`1Rk?>Yp3G{tH@t z%lP#*TKyB_*RRm(OUJLjPpkjfIQ352P@%M>hiyCc=^#ztkx}s`mWck!CT_?#Hoza; zzt-x>Q_!{olJ0v|M*)@6rKl9Or~Re7N%xe`i4oaw@X!inBMU@P3jdSL7Wu_{T{C@p z@ATz9T54#s5bAb9y9Noh`tEF%#LPmmwdx6m454u#s~cg@u$#VEz}?}hgAAYx$e6C zGv-hJXf+tkAB;a|qWP)G=lUDrPr>*2c=I3aA(cnDjd&)G@&9N0XTwZ{i%!V8ipR)G z@JX96k4N(rYCB&2d)dQ(v>yC_;`<|Wv+*Rz$Xt3JWMr<6|LXa#f&ZHLFWqrPjG4rr zLwHa9B^(!U{02w*!;&Q9cnas*M|+2U1`4testcVsmT9&0y8-mTg(N-ekm#8$3BiO9 zFRq($?8D)>6R(fsXvJ{=#~_Y>MEPkPKg97Sj%zrGLb{ZJ*TU<4_3k>nE569@srLCj z8-sbdi>vGEP`-}K-N8oxy6VOT&*&0ggD*%WTCa80zJPl}9k)u9c^Ym|v@YoJ*At3t z92u<{t#nQwGv=(P6BFvJbb=4g={#@(#!Z-?4uXw=PaQj8S~;Y^_}~r zxV}hjJmTg0Y|u{D&)L#g7p$*FJ@J|NE#B3ivt!4O>J6JvPQNXXMg7GXxj2>9k1orj zvLMQ2e5>v9sf^@$I9c`}l}YRIVm_9l+PnUE)?ZrKit7!509Dg(7_7$^zvDm0Y@wp{ zC~IYGc<+CXnK96K+NT&g1lt++^Ikevud58!t%4kvH3WPQ2mQW=P0|i&b+!K!o?t~? z-AZ>uHRO3QZB5YOZwx{+9`X5uA$OhKQ{U*{E=lFoA8pqJAMyADzQ%?%q52ItlcbNT zH4laSeosTt;n^Tb`I%_#40-(9Z4Fy|{>FxS)I8$$`&dN+H0}ytL$$3T=--q ztcQdJ~j{d3#H*0pO=+uGKQB_o-wE0(Q(*k)a`*1C4ZiWIsaw|woxL@qg&qjJ^S%2idX z9?l}y0w)RHrUDy$8#*JdT8akRd&bP$|~5=4co0)TlrAR?8dg+5c1Vk7t32L zi}M!B6`qZ-=jJbzw-u7%Z-qn}w+7@b!b*zexjRCO)r0VB9#~Uy?RrrodiZ_# zAs6z;$##nZo{b^DC{45#Ws9mk8$z4LwofTcw8!s*c-#Svw6reOm6PU>g5VSMdY~~| z=@08;K&}pvqmWr;=ww4ED5F)t!@A|$)X)g`FVC+Ic}N=*dN7)YrIz~_Gkq;oK%wLs zx33OexB7x!koO0Dh(Ft9cT-cH55+v7Cm=6cB)c~>q84L=FLGU@yBg;pmr5%EPY{&? zL3lH_zgiZ-PR8%&Xw5wn$!?r}>+owk>r#5pCP<+G;qiMmh3edXb&v;Y{2tF@c|Iw| z{XARzqzitV$Ln!dd#KSS-$uECeJ0`YHEeO$`M^5!wz=D={hBJRX#i|>2O;yZ?drH8 z+KSK(w@fV>A(F-#+3#-HgdQ5%xpg5HR$~LWN*7umI0=TQ{iaQ-$0hC(bPD&7hi`{F z$hME)7YfyR|>*|Tj_5Ko*e_xI==|}yIP}mLb zjhk<*-?K5EHugH=K*}{?(drVoL?aDj;#SO=Eg zGdD1MZr~nLZcgKf-3e<&Mer z8VPij)F08(NB|JUytx7LQmO4UzEPim*Hc%gX3L@@%@KUNC)p>FRg+Ja10GL(fY9T| zx>D>zcGn;u#%R-IdA`@z5TwLlJGHCxHEb3RkyTqw)ls~`??wxc+YjI9m%(kS-`H4R z59Os^p7Q&pjb6{j%>=2Y(Jv=vlqzeHD~P!Y*$A&Zf!|L~^greJ4}DoOoxnkS9PiU= zyK&uk?LW-2v%r0E0_^fljJ*%I;}c+CdJ|)po{~%^95<5n`kNSg4sb7=0DI?6j6EN? zD<;5x;3mfI0PgAuu=n1?*n_~`J^^-l#f@#@MmF+cl|KXAM<&27-Ne|>0e9a7*q7eK z*oT1I^z@DOV*O2weGYIhodA35O^m$?xSJ-xe&8m?-VWRcC&1o$6JtLC+?OW6-g^^c zA0pg;b7Ncmuzd9g8OH$JS(6mI4BYc^;#2?i0en|cH0_QU-_8t}c zAzbql${!Mj4?4^d;5@5hH&+P0_u`tLJpQ0WKcs#Gz&Wg9|B#0KlMQ3~s$`mjqYwuX z?fj(p{*w*xUr4A5n5%Krsn{n|Y-$GPeK-!P*n1`s-#dZ12S*={!>f=VOd{;Vz?}8$ zAK83Jzo-wJ&{Jjt-< z0QY#3?`_aL_|*vYX>h6+uMmjZ#3kBMU24z7 zrCR6r@YB)U5q0XN=tr*#q>M+GOiz7`n~|W8`t-Kw6XF-{*3kpg8qJg<(Wb$(mEKqI z`Wv@uZBmLv8(#ob>+n`W6EJPUquVMFjCWlX@gs!?cq3p_gQr?_Kzo%nYVnwPr#4Q` zA8#2l4MA6Hqdn%XLLnK9tr`!9JD9{i8p}F&P{XdtK_jSjx*n70MDx|!HrCVxJi)|M z_&nN;tD!sUO9W`6o#I(@df$@uN;}1~>!L7;LGVMvs=Y7Cih+rsT|!4#chm-&hM<0w zuBcau_bwAN5VSOQGfxwBYHRJC0?@opzeJsAsPRahUaHKrA}PVhBoD@R0|u^4yt{yL z6TH(5OuQK}nuowWsw4VoY~^`r)fz9EgX$<~I}twq?Go2x+Esd% zk*>OfqP5UT^uMk=wO>NUs?;!TRidh>$uDS9Pe_Pmsfj4jkmow4tv2BV4VK&lHj(U~ zus2}3u5`>;sw)@Tl4z~I3pA?j4a-znY9p!6LsRDK_@eo!G3_-Lq5T;CB>htYU&28N zQ%LCgbu<&KNVU^MZ>@IJJjc`~> zh<=j;>+EsLZ_!D3G(!9yL<@coV(h1+d|Q2;_J(=sJj-J9JiNAvw=41fU+Fwo)rv)h z^XNTGyaZi`-|{G(x7`z%S6((_T8TRl@YHX>E5RVp5Gb7&@;4L*@akZ_JFuwUx6$7i zXsih?!mEYF?m+$GEtYwB!?wX!;|XZL`XELFDOoNF`a^->s)m|ImD)Y;K`rkd?uVZI z{<<%?UA;z$-}BiJM)6cT{Jt%CGklXLkf=X#=FK2 zN;AqoVy-ZM((E*EG&h-lW_io-Sd)w;#H%lf?aZR=wtPnZ0n#9Ufa z`f};~vZAtZ*}1aY%N^zJ^8Z_|luNQC(eDS`W44$J&E@75X1lq{>@$DS{EYc)=9kP@ z%!B54&9_=)%l(!@i_21L*=pHkdB*Z}i(>hMWkzmgt~<9T_w%{MjOELDPv&pRZ_YoG z|Mrrv6q<@0)+<(1$$`@EmYyqZDI?{jA3zB&SzNHZ;JXEL3-=dx6@9b#bn)xORn}(f zPpxlRT_x_4qa}}(b(Ehezf_LKONDMMUw=L%^mo2}tWai$Hdw1^2 zTyO4{+@I&B7DfxHiznP!0!R8kZ#8Y((me5AO}+GL$y zvarNlQdrVgqLd7j#KFCTrH4viC_P;Ic-i{0>N0OxU0HT{PPtq@r+j|-h4PE#y^!Mo z@v2q)uFO93Q|A5VgXTl#)fR{4QOkNuwM7p-I-Pqq_gwCU+}6BZdF^@o@=EfT=2zse z&L7Gj&X<{YU;KBlPL4H9=fl@F~5HC1bc&PAj;rgQLqPn7- zV!3#J@x|gx#l6LS#Y*v0)@Q8ytp}_Jt&Wnal1ED(FIit=C^eO4m!5^(drB{qUM$^J z)?T)^?5VP)C%U47HhRdaW_>CLDmdrY{-fS=*H=i({HU}-+EX|fyXc>McBeyWO PBzGzQvJEx-pZWR!74%pS literal 41984 zcmeHv3v^T0neMSIgCGKn)MT855J8~8|~$szu3h-r?m0UK6Y($Z*CuIKN7aP?+-9qY&+fpXa5GqjfKChjq$0wEb>{`#hrwYP2b~*Sd>`dXp zLVc=Gp3Vl*I2yn{`G?iPItXj6DZ*T8UY{ZqlaPnM8dHQY_z>TM;ccT`gYMc<+80UA z?GoC*_SbkLTr}5V{0-o`$W!f*fVG#})}zpbttfG?a+t#+-c5MmXJ}|dDMfh*CGJ%& z2wWW1yghyo-rRTNZ5CA9`xMo^ zHVJQkBFa1Oz}x)0@HXzR@K%W~4DNrXH+!(JvQ^Xt#AwhGtqj_=HkYAM(90SK$A4D0 zd7^wdjlOBliBZy7@VAfsXteo(-gCPbr06?#?tC78?JuDVLHfdr5RS0rbs^)_SEyl6 zyH8_k52d%!#b^{k&)=sDidu$H7qCQvc74De4vKoeoD#r$b6Ak31ANdP*_+yuf^YuQ zDFJrvKa&!$M?ycw@R4vc`dWXpR`A{JZ`KLE3FpKJO*AwfUk30BijmNxtZ953O(JCR zH|vG1B8cwQwpa!rGsZ_76m?W_HE7qd3ZPanW}L7LWSp>HHNPh5GERv2Zn--DbVw8W zfvJ5YU?T{b^rfgGG$ANP{p}C}>CSA?dnW-)nmxK#)U~9B-bqNA#mm~0x@Od@UW)weuNYGZl^|9_^+(968BV;9qVLf(m{#N?kz zh$I~eSsI6T{nl9-r$iWuGzp^8SIRXIJZx~YQC`Qt^^%T)Q;BoH-cQ4~E(b=_8-Moc zTI>TDJ>ZW_cuJRX$}-?TZ46lYLK8Di*$4b*@&ooh=%CLkPvNB0VF6JaI)VQBJ+Bg5 zyWS>$0489|COTj24~XHQC2Y22d1uARCJTKtZSs$J-=G*~DhY?WR708&v_zOm!gRy0 zUoNyqyd?_B$ljKxAW1SBHkbMKNKoug6H!lMY}re#L3@81Y+JnA!dxeG?1qJ#NZ{A0 zgFXi5=1C3BV@UT}v@Hw~he_tm@Ulk5)f0O$=t004r?imjB6vo|sZ^+SR?vdFDexGy zm|yp0#yb&{pDuKs^+{o4CeA;dnEVB1Kkzi2RsQh9fdOcXYYCxDX9H)e{1&U$A>zQb zqs)Wh+ANs)o1?=Vo|^ZHk&z_~ew8pEuv^OGVvDHblY!0Y%$?F*bai5iO`gfyGfuU` zsC4GjzI4C|-G!M#2Fz!*%qkFkfa2vFj0E8SP)wi?#ssu;cL&bdkuk zn0_cUTIEL&PMLlpV2%Kd_+^hW-cd3HaK^PwKFqpNyeGj(_agpUF$dnLXN{yxNHdBE zdh$AQBYm-WN6XDjMemp>q8T^J+hQO>GDwcBc|JQnpNAO9=_4)}z2Bf{K+DNQwr2DO z*tK3@tKL)Kf5S@$0vG}pEgeQ$I;`?*JkUV1w2V~BFR9m^=09M0LF9wI(ejc-0S|$E zGU~3v=@41;Ldk3fr`(8UprcXgsxo?%u?@VdGDvZ%3SjQKW_zv|6x9z-A!I zyx*mv%}hQM5p8Df`3ti_75{ZMGng}*uYs>mn2}X}nOiYaZhCJY9OIlA27Q6;aQ3{K zvHJq*;$moe@5`uwC#BiL*r135SVNfOhq-=y>AlaO11ZiQnJwK8tqxSm(nkzv?~~UX z`#3suh*u2f#C}52A%5iU5U=9%FAB-|(getL!2hAaH)pSXpv8U_-j*hY6$=bA4+Iu* zfV`@AGQ&ds7y4#{3I*Vik#{3ta^yzpeZ>e*O|yjePQfA^TMCK&o>xKpUs-Rg!XITf z;LPt9lOda*`RxEYz1FUIOgbw;(yj6NE5M#XF(&B*J!54%VI8BM^h$L1`!G8(Y- z1noUG`M3YU#eG|hNh37D#I~Kd5oVw7Z)r!@jul29wCIDC(SW_1&!W*c-Gx)9UVfI- z@i+GizT2P@97;$#MwJw35~)TBYmqz=jkL{_>HLdz-wM@V!Rlha3fq+t^tvnMON?M< z$j1k|2@DxZweKNd@SoNMfCRHy8V!hj9az@<&3!`iWPDo~wDkS?7qs(10I*+0AQ(q` z2-+<66O$`8#FMCX^F;K6y(xTX=QCiF|MX`rk2zNc2I3d_PZ2pr2`XdQMDb(T`KOqj zf1)G+cK+ciy}^(Hdb|2sU`6M|LwKh}ASGkZI2;#U8eF&T*?x6k!0Kuw9bZ`7n7JkT(!hC2rHdM-E%g>hcaMA;lh&9Y0O9gS1 zO_Wj6Vts%~_7ef3#9|4vV$}AblGXNc`hD21l9dA5Q$wJ&5y21o<7P!iX>Q|us5}U5 zh68B;!g~gt))=L7X5u)j^$X zkiC;mDOk@T_q@Tg@$RWiNA1v&LMiJBP+HF@DZ;s+g1W##U0|TjtMt!v`sZ8h(IEnv zON7?MLprMtIm?Dv&JGG>RI@23NYOWC2fsiT-q_KIy-F=#S%D0Za>ZLa+^@_E*#Kewj|NrPjkIVbLHXCJ#SvX7QN_OVavp|<_g*vA2)4@0By(G$xAuGfOy2;d_>6D{S11#axp3>9Dc%?SPu;&xH(1sO&^Wf!kebZ^tp(C$rB}mKn@If8vvp_K=G!K1K zeRsHyQd8^sG%B;_0W#ci#;9yZWX9DLbyr3WyP9JKt&v z!?`22&BA73V5-iFM3aa_Q@_ykW$ZR_AThmK6R^lH(W+h(uwQQ7(}ReHP1t4u`5uG| z*(TfXBf8PxN)1SbZM+M@Ml?8O;0^39GETHZRO3<6u|!L$)sJuXOg>rc=q1CCu-)sx zDo)&8>%S=IHw&^nQxK37p*(5R6gYi9U+O;-?Qs2^9~iU=;MvtzfSVz7Kthmtfzo)s z@uv3Pw-7Vbl@1c@(C8s3Ed+JZCL4g!)LsLaFKTiAfMEZl_3NG3d*U3qS!fk6T1g%9 zzzhn>%$!~?}tC`P>#b~Q&yVIIl`tY#YQ zuEF*h3VerYwFkO*{cWaL4ulj-447hDEi&+S+vFw?Cz_CW$MTc`fvr`a$=;ah)8KE; z6eOf~GtW_^g`9Mi^5)->`{D2bhwT%XNNFttdYl%J&h)T$9F>`sviALGXQZ@23sP+T z0c70JfwGt|-Jl%_k8s-3;(&;0_xMj|1uWe!(TU~sJ22aB^h&GHuV*Ruh`&gXmLe77 z-ww4F!op$DX_ZL6B6kz@7YLO1f?1pG{X8=RlSFPBXrrNDv;1c!=!8yOCtGv4 zS)#B#cv34eP_iH0T5)n}9TpRqW2rp*uWJJge*tU_%+FWKb3he~HA|O7pjw>85OmTi zivy$_+25W8<-ABb!M&3p=_$f>g0phJ|49S31(aIC;gMQa8lXWXeo=N!Ioq?=uz6*c zcRlKuW+Q33{Q=9Rp#2hz2f4Zb*~4NxT-A)@A%ie0V800a2wE=I1T&XFKF~DhGA&@~ z%6NgUyND}M>I{mV$j0saZ1k(c9O!VYs98y*lNwrald2V=RMt%=)5gMu+#;+y8bF!} zmN^;FcfCUo3XnTvrxd=9MwmY6{*qO`hsl6UmSto$Iqji*GV03QUGE42Sm8tvLGWN_ z5O!Q+nJv@~JAjSGl<_mPiakRNm$`Ivdhh=Mb=N)#+oh&=6>pa*;7~e1HFCA@9@4v| zhb6nZ*&VU4S&T|gz-`G7qtYhw0VG=Cs?p$5%nN}bLScn!`U903TFpp)q}70yOapmo zBxl3HN@Pg-X*e|~qbQ>n4J_4=NbF$@XT|}+Uq`~D`!w^%Vo23xplR=c?pToHK`g!Z z9y9@=RenlsVNQ-@QW{#YccmK(bVgl(;dD6c%f*+Vz0c-aP6Y2@!)UWhXBAjV1d)$? zBjP`U=tWKz5HG^WAxA}#INEBljRjG5{3N11dJ?wS6CjI^&bM^I&sfa3pvH`!w6>n5 z_21Ho_21s@KQjwq=ibW347JU^>mmPd?g)OP^-KP_r75Eu^IakGyp-A}V%QtA2nG{FaG_gi!AjcDpn-1r;4T2}tv<(zjW+Li%<f-PFCgqfHrjZW)1D$wUu%@JPR^mB0|K|1+1X&b=vVLFtRM9h1{}$Yy39CJjXRoR%RP zVFsbw!4KyV@U+SI^SVKAi*#rv_9Bjh%%HPH0-A}4mz0{CuDBSC4_);kr) zqL6syB>2w!#YpmGJJ?zl{^|Y?;%=q<6Mp+fi8&d29-xIorXUJc%00hg^28?Qu?}os zJH)pKI>dME9pbzA-h+eDZ}Isau79WNfHpeBUQ35~S=%9o9@|VY6}t;$+`qx7P&Nt% zg8P#o{N)(iUdjZz`dQe9>v&=uH*|=dp!hmCz5xzz;(Hf3{uI|g^L2{Jeg+JYVWkk_r`Ii-ScGJyymxoFyWBH6(zQG6!^<8)rnSgV9(eg z{vK@J2k8&+y&r6GlUDo#$o~q}pmeIR|-4?U>+!&auc{z|!$%LRwLemf3%aH`K{zm}

    K+)gTjNQ-T0!chd(Skj4 z6Jq9`_Ke+M<$Z!vKna-)S~e6!OIS2f(<0J(Mkz~>@DvjEn^|;@m=e${z|JjCp)8BA z#SWb0&HaH2y@5w7kfm7P&jJ_0j`4E22(IlS!JXFo6d2xW17-FGd19K~o7b_m;hR5a z-lybLeAqbS{MJva8MlponjG!1@hshO!v8fsz}1cm(u};3S70~exa|EO^KV!`hoFTjmdn##01=Q61d*!QAD0% zn}PGwppQJJ4u{-)8VnOcoscx$)gxk`UO;UA*njASCVIX@(=~Juxkc0?AZT0H zrc6_kG_-9axlrh2!9i=b>sZaN2_=rtzB}QXv}=UOD3<5~m0Aq=H#9?7Na`U{!&x={ z-tjm^!A$W)E;1xYl9ux@gLKM2c9Eq}(r-G80~(SeJldu4!CnrtRvo5Z#~4lqL96_w zKQZeHEk<36?=~7xi5+5C*&*t&Yoa43J?MuME7aK!qSy}H%WloW0g;N(1(s@C*+z#p zKos);W&#Iu6U~sxADdUjT$)LeXSMKbwk6C~bUlwcz%?G{&CHE_nXY0IN$ZdT z38B0B0G4XNHBU8=$-<9s$d$N1JeP|oZWgsnyX`a_iTz1b$zZJq1VG zwWU!|DIY?UoP$VQ=FjXZf$g*LlEF;`mya_4!3dzP$do1b6(Jz7g?+DpXBH_Vq7P>7 zvbk!vL0zxBPU?cj)oS^5R)hRXIP?M=KgG41dLHNX$VNWE?E@L?x1dWJjWyJ@vjCl5 z;N;j%PWA*2EWnb9Ae-SjN<>;Yk&u$qyk0tMaH1o8!~83;n!!D=k1XTNm5djqf}*{B zuohP>43o}s^3$Dcw(#azt|nBvTBwDTm&V6hwF}ixId~e*qb{cq|S_rLz#VTL?V~it-rG%yHa$TK_1CBWwPJ-WE zK@ij<9%C);M!Q_X+H@W1w9iSQaCZG1#R#HISk2?XwT9)kaG$`_me?;!O^T1e5BS-J z9nl?w5wlYM_W$FChU@^&#T1tjpfg$E*4SJJY4vSXx=!=Cvt>w+JR}IWE^E5 zjiaOQ-SR0@MwlNK`x>?x)mY`n_!m>VdOYUsE7bOq#O+^H+wV!-K2B}Fg6skI9mDCr z$E$HZO5EO|w*QcSQPHvU1Mq)WZGSfL_^+z%&HPKwu=&-f?T;lMzd~&z;EBJj&qWSY`%s+8^huZ$b#O>cz+n-I)UP%cVT9!N6246G8L%4loN5PV6bo&K+ z*G;Tr!bI@U{$!H$D--N4zre;&F6)dY?p{Cj$KjtI`$b}=o+IGfPX1KG-Mng0qeI;g z;|GleO*e4O_|Y#MtnyX1 zgYk=_r@H}q+UwEFg4@x-of50cMm5alo0)jNkXH|*nb4kwxB>9q>TBA*$ZL7libn>% zSDC@9LidGc_)ChhybIDD>Q;wrb6_e}DvmtTlNOi{9j1Kaj-`ZFhE_0MGXG%-=Y^yo z_C;UkRp`XHl_%mxmBbB;$4K0id=_r8>5)S^%J%t~+_pG|8M~jRYU~@{(Mx#rgxiuF zW{;s$VwBwy%4Z$4()|qZU-9wCFd;mfEt%e|pgnUwozv(5mj)X0pAcP5Mk7`%*uE@Teft{RVA#+ioZgl79oma{qEr6N+TdtfRxa4A!8*T)5S z)u}$lC-Vp9(gU*v+~gmatzqw4_D+xG49wQE_e}Pl#oni}cLUx(2CHP(qy&-@NJ=0n zfusbI5=crQDS@N}k`hQtASr>Q1dhmDQD4RC9a4j*s=A@x zIkd)I@0O@W9aUBB_BuAyvR<(|XZ;9ORg%+FM<|l78*~B)H!l3LyWW(u&gQIjI=s$n zJ3{e|)yy`yUELV6WUr(Iu9bjsyKrq}DKbbN$3|z7Az_=-<7x1W(6-Itsdv|JqSnMO z%PPx~ii*|Cs+O->vt((7SY;EJud<2D)}s40ZL1#=ZMId*MYLs`*V^o&`4MrAdG&+w zeZ>cdlM#&O6-z4Ch~@`Zn^&(`5l0u~mabkyR;^w2s9068ddc#t)wboL zZ4`diC2NLSBzFK}&h z*H<@e^BT4yR=6ALQTwECkx~e^8Z}txkGo#jfDs-y{#=*Vgu#9bz0Qq353g0nDs>C1 zof~|chWC%FjP>{MNTl+tieFBg^^#`?em~=KdYp!B^v3MNYpC{lUiw9o`=5|#Y(0XS35k_2EN!D z@PEc?cS5^0dr|g0!jqt175?qGDz5hog5dKKe~)vMuh!vFh~8i0aXJ^p5hH@1(^tI& z*(r)A2@*`&Qb6YJCmj;h0b6UR_SHIJzlH`+wbN5Hq>r;&@YOGj4Z)=_n;H883|L^; zfEBLDVB9_%aVVbGNaBF8kw9TFUdL7^{#<%TLwz;b6=b{JUFWMaZ1&YP8ocf&K^A^v zD5ZBnsB&&^gk}ZogEq455*Sp~Q8)ZLjqQGjCbI^b$lTeD^UQ|9hD}b%u+8I^oQ6hh zh}>TC&A4%%1{l}|$Hpxq+v5!IQ+Zd>tmCc;i`o{$#XG4OW1oB;3I&W)=cRo5HhLtY z&J+b&gH!N0y$!Vx+05DA8MD1J$vD}zkHi`Cz9G?#NyOva3KushTTF64?%&LhViac@ z91`Rl)#cF%bAya7dfRvVnvBl|K(xhe-|(sbIF3qJd&> zn8-{)oiC~dugh6mtL%#TK$;ut7K@v)-80#=!RvI^c?mK8q&&_<4UQV@d%|a+b{#HoBY}w-BV7 z29F^&qj(+FAo{B!6@-LKz8Y|B{j0MjG=;>sx^1 z;3!!3-$E=$faCZmSdQI7EbYM2ISQ7}TZrWnaP*CW#klm=*-IEWG~XDRpZ*qN(F4b< zQLtEVA(mX=SUC!o{kIUy!@wbpf~D~mV%Y~AM@GSN>=t5a1CGv7uyo!+EIowdnUVdd z{}y7&1POB@2bx`fs6q497AT7>p>z3Kr`a%Ebx{>rh+@ zmhv&g(gX~5x-f$ zk&9BU;2FzUw-y-Iqqr0-V;SqV149eSA(Z`yx$R@<)5n3~EXqZcc3|lrLoEHk5Jk~F zE9e`6ngzaVFgR0g2jbv_6wbx$Ttyb34B`= zEZqv0!?i^7wJ3*4E@OygH!$oc94OtuQa*-QjswGa6cXIdF~mXqyMZHu zlKC8D11z0m80&I@VI_(cg|Lm~o?$&ONGL5R>mir1T&E8MLmNsLN;|N0kD-pjz@Y!O zqMtFgT_bQ*pwy!1A(Q?wl*<8NIF51=rJQgm=Mu8x|H?1^%ou<4rV39vkSy3^WXjVx z)XpEd;;peu^rN=;k+Dmq&EsTGr?7!&Q!e>9dekb8A6+tf%40MP3C1W-&+sum_JDL1 zJshOcjH}{(>Ydx@c`uizVVl|~u8Q|@d(l+oaylA;X%iluQGkHoHBBUsI2qvSs!jFI zYCZt{711cw!{!~|d2s%CDxA^abk#oUV@3)T(!ubiL^vGMAoihHsvMGvUDbn1P#ttF zCO(MftM+ZEsqs3c*i%r9+sMtZ9pxb|^ijVG>VsXVQ@`T5Bi<2p>Q^>D6|4GuJ2MPS z1pQ(*!fZ$BpjrsphuDg86?;xTHUmye6*uDEH3`6y1QVj%la z-x2E+U8*#d;Gr7xHF8m7)Uf`lh|pgR z2Wk%%)P`V*1t=~^HEzW4$J%3}i1*WLS`*XeV`F+y{Iz<+Chi~c*D7LaB)wx2v012f z3c1+$_-oQ_74OK##$PLb7#lb8wF-6c`PF!uF`ta%6JMv=0QW)GiD4P>dYInhQ1dcgBU32PLbgLBhXqJmmIBK1Z#^S=T_ngEkO<^nFlzi2s87 zL0{bld=rExh>BS6t8;pg_Nl;cX`R&;_RCqXpd$p8KBs4gSijZnX{fKmZ!RBlc--u7 zIlMG-1ya=_{nmO#gJ(%~HGcL7n(^78c*!QeCnfMnB#052uh&_(0e^G=0`=aKIX+K)krzKHu5)-7*10!&8oUiP(n9>? zxX9tHTeLNM4t^0|@2+ur)kws-ft0~eEO~rhX;pnqgFC@fC~Ji9RJ5a?#dolOz+{?Ki<__de&hzDc zB`=uwblx+0|CINgyg5bXMK2b8uXuOK-V$%=&r5GFTUYkK%HAylQ3KWmd=wbJV?1FL zvR7sQ_w1i!|0a80PEF31obTkkkaH^Mm7EJXZ|3|m=MOo0(+txb(=yW*(+<<$o1O*l zADGUX-ZZ^sx@?m2cIW*|-uT6Tzu25Vr|6qS-Nk<_)|Ff?nOB-qYA*e0>90%wPzpv^ zpvv*_{p|a4zn-hhv*tgMKal@#1s4js3&g@_3STq7Rh(Z^QL?Y}MCq?e|5CcOELgU` z?0=V?Eqk?0F1u1jgF!+_HO@Ewweb;SR(3&wrQo{-FBd#h`29j%(d|W9MbnFB75%p& zW6|QG;v$+H$?jOe@q*5R;==O6b%o~(FBHm!Yl|K(l8U;EdWr;dt+~;>&m1vF%~{2} zi(8716zfYeON=FlN)DHtESX!npmb&F@zS=^uF~?d^P> z^H%0{=5^(*TfBa;ytr?1|KiAEDSvx@Q~vIJp+HlhEzlL{3-%T4FE~(eu;5TZR^hZl zL*cB#xrIjxj}{&)Y%4rjXe`RbJeL=(EIL`#UUa^wv#6_RrMbe~Wxi;>WbQGqFRm_j z6*m_5755j1i=)NcOPWe{mh3KRDbbW_OZS%^C_PwuxO7^Xp=?&!g0iD!&zBu1yV%b+ z61m0#W4W>2c-DBqSdnebemMJL_9fU~Pqv(0o#V=>&1uY$a{6<^Igy-bj$mps?KHKR zw7I%meQsuMR_=k^gSm%tkK`KiX64PzTaahWdp_@2-toM)ypwqai;EX8Uwn4)`NbC& ecP{Q)Y|USr|8V}g{Pp?ZpL``Hkdy$E!2btt@imG7 diff --git a/third-party/mimalloc/bin/mimalloc-redirect32.lib b/third-party/mimalloc/bin/mimalloc-redirect32.lib index 87f19b8ec0f7ae1024ff508a738b517c71f11aad..45d7297d8ebd45aee3b18c8a0bc71a691ec862d7 100644 GIT binary patch delta 118 zcmew$_Caie84C-q{=@^5typ%kXfZUmP1a&ppDfR6$JE@mIiA&!kuzxNw&zC~7#Nr) z+p|bd-ojP{Q>G78GC7`AYH}%iENAa+`AZL>s>~(}a@50ANl#wS!2wmd`7*~`CIFm& BByj)$ delta 118 zcmew$_Caie84F7UkK4D&RxG<%6dC%eCu^~*PnKu3W9qBk9M5XV$Qe;K{r^!01_q|d z_AJtqx3Cq#l<5PNOpa%jnq0~r%ejdy_Sr+IDznLg9Q80&(v#P7a6naVzRWR~2>^ga BBijG~ diff --git a/third-party/mimalloc/bin/minject-arm64.exe b/third-party/mimalloc/bin/minject-arm64.exe new file mode 100644 index 0000000000000000000000000000000000000000..637c95d91576043ff83cf2f0ceb5a5555c544764 GIT binary patch literal 20992 zcmeHvdt6l4z3*?&0M3B;h#;s**x-Xl1m8xC#yuhmK9Ine=jjJV*f1k7gENDQsTH(I zNJ!dD;-^XVocicF=Q5nO+Fb2Na!(tuFB02)&^(&7rwv5C!H_nl#>gbGbHBf}_lALJ za!=ax$GNxr4ilXO(rCd`d_Eooi3kT@AXuB9d&Yr!|nF?<;|+>ZE(vjx4dd&sa)r&RCD6vV`u2B|7G)! zy8Fy;956op53W1#KI)eqdUOBV0{(7)uYmutzhA)L@4pQA^qx2Op9AbOzj@#Y;O2*V z1l{I`(ho5G=7)YO>LsoUC-c6MCdK(gmF+R~-5#~tpt(e86lWTjMH#@%(y&iv15OZS z1!`+$6HyG|4;};3Ayn9yc+6DZY;@B50;@4G0MqFhF4C)ZqHZoU5%uyw^+d&DKt|$u zl7v_jiivL4FxUOtQv@GI_EU(KXrf-1KF6=_@T2A5Szi{2b=UPkNjx%9d5*Wz;dc=I zj0acnxfP$;_(VJkT5>cpI>|DWfo>aQGRRI&|JVsohLXXWup_wD*}4+bu+wEFUN# zl2*Ka?ZK|LD{)Hba_Oly=6SjFe+hnXk2eL+Cz*r)J0&Xk8Qwpoz#nwpk$ica>+}A( z;LY+I{QFv^;JK$w!3zh>!FSS!5gFMhfrlyiB-=zS{`623EBgUKrT@u>Q-%uZI z%**3%9_(u+j3r?#Q}CS>9_uHoMpu%CjlF6(14Uh0Ys{21dIG~CW|^ubndL(d~d zTeA{kIk}DK&R4^E(wQ6SmI2>c9IJENOtkQFsT902i9)O&>&E%h!~rDVhSJ?_EkrlA6wPo2Wucz^z(mKdnkTnU!C1-oWQ?(edg0gICdw83rYj+T3BT&I zK!TmM*x|8VKsbdOzxGqiJH96kqfBVb~R)*jTcE7!D`3Bq|-5QrtSL zEwj~WQbLO)OKZazI^I4`>6nZ$+U8ThwUPn|kcHL_wv(^JJ}-vDix}4pKV6(I1+VX` zrjoZzv^6`5>;s7FM@>yuuGW~Q}b9f*;$V#q`21aq5YsV zM(AMbuc>EUlBUsp4 zlHyvhxOS`sgil@FT>`Rrul5!^&Wvc~Vwy zT2i1IbUx&a-KNBWyl~k5(OY>z!RM}|KsnlYY!PGNf4TIhVr>%bj`bvDWdNV;F^%LSZ*m*!});gMP3uQ zc^b~mIIE7cX*d^fih*-zxJt%Zbh>5@R|ec6eeIEcsNwEsT#QcFrQtRKhfTXS?imeN z3|x7%j{6`*o0F1_=h|7;onyWnvGhhP1?CVX2$>~$h%ODe-k|FQUH7cc+>|S{)wY-H zuVhgm&FGVwI@IUw2;QR4ty&*x582;HqX5>*gl?>-DJN*F0^EsaI&lTEj0Zjg^SU4Y z8Y{|lY^p0J*J_m2nA{knTo;pzGal$ch^(kAB5p=CHHycch-fvrU!m- zE&R*{GbJ?arsGV{bEXCDy@=CaBF_GRws2llZjJKI606*Fwm&Z_5G!@&#$sN@-qN|G zUD9Bd4tecBP7Y*Fq%$vKO=X!fqcFkw#gV~vB{XPJN7JnT0>JN5}>5aR}Zg+1+L z;QA`0KzI(XPm|jBqJKBqgkB%)dPR(@uerUqQRrK9aH9gQ zTpbg14zJ=Xc?kSR2k4N!V8; zi~Td(;}CLG|CYfeDIK(xV?>ETet}%i?ra!*7IXDOoN0LP{Gr%y)1@Son-Ei$@iwvk zw~Z6~@vP2h&1cy2)pTOd*D|?tN%vpEi&(Y~QVE^;MuBS4h*R{^Z9+~&e z{pnFbJ_}f}j}9W&zKb=X8|#Ic&jGgS11;Du$Kf3CkvX=#Ml#2()%X6$wRc^FZ|ZQy z=)Otm2&O6>S+=x^b`@3S5jWm3|> z;UcoXgtNktT&1ILWmfA7C2gP?xGX8Db*(gZsE-VJxnC{w0{*=PF+m=W`4s~H(qyH> zC0S2jL~K85%Q$_%WQlXeD4})X9G5$9E$aCl)qG_Ki zO$oj`UFmplff6#=mY!~bJgK&&t%ULH+zx$*&Qoi(^OP%US)C|YXmM4xGAG>Q>dc`{1ICdYrubn6jxJoEo^7mw!rS0Qc_%D-T2HJ zl(*C=9c4JvTJOXdIMdc32gM^ka86HP8OK7#jo@)~87*`}u9s3NumLgs0(iDTE*pH2 zO7U?kC{CPhAIF#nrSUZ{BfdGt|Lref`|A=Vyac&xr1+upVIAx?md}jlX<1Ih&bvIP zZL?d)!v1fV>3Exs0)M(q3H@=5?YVa_XFZZNZhTkD$&C!9l#_F$F>&MbrIWuv-hJO3 z(|!RtS4Hl)fc$d-x#xlubNZV&i*_Y+F6m38CAd>s^h9CS;8ED|iNf^30m%7|LKD~8 zXkzjhI+J`YeByTK%KM)rz`hEcUl8lOK78^?_!*8@zUwez_R)n}%x2CU8MAc@t{<~8 zuv3*36ZaNfKW5oZkHb!*?*Nt})~2S>@l3N4a;4JoLBt7R-hG6%=UwxZ_G$1{IFHX1 zl>2SO^q^RusQu%06uP>Ga*g!~Yt!-f@^GG=j<{S0dn32?V{U%3kwVYRoN^#Jer#+1 z9NbftP~hTQQNgLxamRSh6#O`B4)$ZdBuWa_w#1>eZ3^rhf=!d9#MAxAnOOgOupaeb zZR<(kwGg~Iuzv9RRG_a<{aBwq;WMy#N*3pg-Daw3LJS$}6Y^a`0`9Q7#oCyxow@b9 z9%Ee?z`B4O-fPsMuL zkC?LH?AniYo!16Y=5+z`36*|Ng4rK2zf9 zLGZbXu_s8K%lpjJ+NVxbIyk@V`I-gqK9bunO>bI0HH!A_`SY$LK{g^vqtgV`iT$X+~h0j?%mWxUV`?3z!lkLpw_^a{aeyhjw zdnHsJs|3#>X2o3@+Ic^aLY4vBIBl+3`f%T*&sD+ik0a}3n>M6#C?xKKgnhelzRzG= z!#>?6-9CNO#2kAcbb{A^(l(q^xF7Z196-DZTUg*1&5v3<9>Tm3ybD=Q4JYdPphMlT zLrN6Q>$K?nQg0P#D0&r00&@O`HCPD03FlLK$+dXOzO z@VZe>oeo};+s_*H<}^`%*{DlP1E-C$Z6a)!)c#AO-h5NwXGR(Ko?S-S78m%5UbaN| zSln%q(!qW;;_e*$E9C2%68PbNwomiaz(0RspG0xC85+lSOW+yw;d`+Z+5FnSwQJAr z>E_oik53sKdWQkKK4GGLCnehVUBKr6`QG#-%99YEGTP4&?ybzU_zvKp!!eR>3RHqF zh|kx7JA^oDNBJ~9h=qlM#xW+a6K%(UJBrUg;`2|aGgOEcH=_Iu$}i)?vBB>IE|0Md zh9MunFYCp)KS!H!ug7v5G9MOrjt9J((di@G`_SHm56c!wLwJ~O_}!j%SIs)^fsVL` zTa5E!$qQI}s!+$A%vylC;+$_2?;&K|^*W&|uTReTQV07}N)`7BJ=A3lY`}Qs(7!jO zsbqyd52V9ZG2h8y4Xi|af!>$zqZXlj0%KtwXT5>(njtfv;hfl?wIKKZ6L5+|i@9#X zdSC-R=E2$b%@#iA^-Q=G=TOs%*Lx9X+amh5;a>k6=nGjEm!t18j4R{AIc6^4B)q%4 z4G{0WI<$9}I42@+vERMmBg5C5;FAo{@Etqb=M?y1ZLo#kdKwIJWNVGSz) zeG2HAX7~>H)R2GJpLd}T!;K6tJuD%0Na+==yK%jf@@{nPMqrsuTAJahIQtJJIt?T;7$T|91%aP6Z^oIZ+yTh*o@UqExj)~T^rH?sQNMul$Kb)dB4s8#THN7Xvi?3<#MP<%T#hlP znHC+yU39mZ?B9Y;h~0&LiN~fVNs11<0o)-W3+j&O0N&{c8kQj*GBBM5bU5EUg|kl3 zY9(O{3uuSyh}l!oxrS_)ONZyqA@F$@vh!Ti^w;JEzr*MchkN2A3OMnuVMS_2O&{(z zoOpk+1@AAc?JXtP({%8Aja1+Z=JERo%hoRFF_z*^7i99gir7|*g93%|N@J$g+8P&c zZEcIAP>K{cfY`Rj^7}4Z*1&hMhFI~wi_dZgZJ7gmc}+QtJ(yTSo6fQiS>IUbYr)yh zNrQ{I&iART6W>j7ozD{tIR?XeZWywEMBbZ(*u4XG?8SWDjdyj2wKeRL1@V(|?O->@ z3HvZICtZ(^1@|SEGpllWP85I__}tF1y&3joSPsbYjpildAwnL^t4kS&#XGBZyz~5_ z1M(n#_px4lk0Zms_o0qG!`5=s!Ts0cXT;V;%o)&q&^zhi{Wp%oAB*=PlQ6C`O$jB$ z(7h>A!ax`N$mcNVb5s#i`m4$-IIq7n)Rs81t#>a4 zMzl+MJNC*K@s2&Snfvk?88ADkS#A6t^QUc* z@yqMm{3CB=fBQC~1v=yy;B|y;&yc7^KrOzt?{qLfl43YNAG1u<-cmR)MmU!rgwS#?$_4+tuYZUJ&dG7N0Z5i%TFrQ;PGx(e$ z=Cp(Jr#`18E|+j29iDT;IVDm)jQ=F1j??UHy}q59{&MMXe&n_4 zX{=p^W=eLNhUIZiMR_7^WqG!wArF`+xlppKTVbWG87Qx6F4)-#`Plzrjzr|SHJFQt zcf5~PLPq&9J?~Zc>U!nqr>BteZOO1-q&(1}$9h=2r|wDUiVI}myt)7%o-->Uixclj zyRq)-YxA+lwRyKGVZe;{rK9D!;W&NzIMUd^**K8@lI781pK%Of4{>b5eE7qZqLp(Z z%2^TRJrU(?5#^l`elDUsE26wKqCA=RWGNBzQ^LL+_gKgUgn4z8b2IW) zWIjNy*VkRnOA_)2@+xTiiITC8sp0%Enm!ue9pw1=`}5aX;L-zH|Hc2H!73eoU5CXw zjMbs6Lu1r8?$JvBs^h2PGJu~b9locxzpcY#qx#3(r_q0^(|<>Y8kVR^uNUidPu*{{ zXz;6}>pGo5m7&+~*I}&=SLtxA&c9u+|4N5{(&5KCwCLl!tHWD$I9rEHb+}50MLHbR z;gdR_=k)r^IyB^5rnk@6;RLvgzKhd-Bh- z%5(=B_$%2i_clkZt5V+PNEM_;y*OWX)j6tFnZI5vSGv4vh2P_CBri6m{4}^7n`>3s z?~y(As#~sd)v8P7nZ8(=|Db)QZ>d1aYF)j*aklL7g4QF~I=t1m_Sf2nd3aSvrR;aA zT9*os+poI)KBnUqm%Ca{pXp0yzWj}N2c%`0AtR3~XBa|@FUUKpT@|vsp>DJ4&HMs- zFz~sm-41_)R|PFU4L(&)uXDM#K;H%P)8#4%?p3#YU4E8g!TjvaEll^uS$-s6KU zAluCO4NHgRuk)Z8(Ja9~NT@JsRDGOZWAE@m$q65BX_iuOkog$ds>UMotgg(Rqo+akA>dZX4>T5i#oz^NXrlXO%WP{S=Gjs%JiuQ3zLAc@@_JRbqfU<)1eP|l+;V-b zqeA_R*t*=ql3Y*f>}~LCHXOFuw9O#awo_Q?0EB^l%vw@2_(0&>hK1YzVh}S^b^Jk7By0JL)m15&1ktxPy#T*ieT& zC+wM>EkcB}{-V5m8H@5+21pBPoh!S-Q_C4btyjGW&B`HmGWXIZ3uJ0=*SI~~!;i-Zd7B3XJRNbLgGzc_16*A62EURvwAqWpknf8ST+Wfe}p$t z5RcCZYx((SSnOk_*#`Vj)Cii*s;mbQx7UhxHo9oUACJ4XQF!4qNr&lOf?kg*(WQZZ z!-J^_GeJZsm@Ohs&Zu+Ls3KQz8&U@^Eb?YYMNLC}Y9{ll61+qN3r;X#?RH0_Pd28f zDzcHM0eRc&!Agv@f_~x|8Ku1LakO%79*R)icaa`7dh1MdpypKP$joAxX{EZkp_*3& zgj|i56AeDuqK#L>*jTLRDssCl--E2la;6W>xO7AVjKM20!drNcEl{nx^@txzv_4K0 zB3odg3ai%2>xvo+F`CblOiP1kxrg>{f%TYbpAg&4#$$6goMyy^=znoE#DbJrf ze{MGZESxQGa8-CcK2McjE>V4|!&~8;Ew6IGSnE~S7S$b#VeJL^a_Pp^_Iqq4`Ep^Y zTwJp8?!s01tK>9WDe7sn<$DV41sgZnWl)sZitP8v8&}J=qWk1^g+;4o%lY>fm*kh0 z${S0-tZ+l|`oer*3ybpBZ(3DYv_@Wuo<$q&^7_IJg?6yDZxlo7%nI|-bG5u7za*~! zylpEB*B9FF!V-BS_^^gdsL3ebQ;^R{9@B<@dG^AMMXXHT#v*$Os4eJ}#ppl4CO#)pkQ(l;PsUu}ZwhCK4BeA{|( zgQJRcMGaRX2K9Bu0B)62zqSt+Gr&mMa)#K-W;?Y#EN3$$rW>U9%p8Ls?$rNF9;N*H z=a?lcHN2w3&kX)wB+qBY`I3kNUGCxt`HkPg;@t}2evz7r@mcmI2l#?+Yx3;t`S+%{ zpCjWps20?VXX*7Bb#7*HFWz=k8Z?8J-R-dtR=g!K_Fah=Bjj6DK*IzNC&?fj@8iBiC@EeT8 z*)=*Ujp%FF=q$kfO2Rlh*zVfzm5*Z_-4F9oq?S#%xEsbTVB8_#`1i*;-9pCk-O+n^ z!K35gVLatTbD6)!p5|hV7NB!B`^0(L_&v1oyJ2b)U7$C9A3S63Jdjd3a81lZ!wH!f z|9MXmm8bPJpF4N1XHVAoX8v>T+|6`I!$lx(hH8C#L3i%eF(5gId^SP!I~RREffV^B z4AS%UZqo$=2^uq%PaUc^N1unvmcHhw^ES{g2mP6r7h}4$K2glmOot|bUX-Ii!@+*R zAky)c2%4?bG~ofzRN>RNw|mbFFoCfq^zH50GYjQ7Qj$6EcK`B%fjT#xlnGHVo`LC( zQsnJX=&y99p0&LA@QYF1!Uox(o5g&aUzI)uuWBluaGw~bXU_~EK7}Vu;GH}hFaK~M z!}FA3Jw8nl^|7Q_I1Zc9XA#v~`u2W$p8LX0bTfWT^C~_q=*uuZf;JgtysyXjpPtWx zERY-JG!{kZkQh-1FUxuG>qV>?&l22UDGFq{+rZ#%5_g`xz-8!nlH4*2xE34Hvk|z} zzzqVYMD%q6cL;su2wV$rO~BPh;C=vH4{+rXxR-(J1diu|@jT1-YKl?&Ed2ZZtY(wH z-_L4x_`kiMT|quSHr8GaE|f&kLStaT~)4%GTby()~cYG27E<>7Z(=(GId9V zDh~Yk-zBbtEfaT}_@5%~&MVneWG~#1KX2h&?Ot#bsXJVLqDQI9ui~Qc2_Dy7;qaIF zD2Xb>nVM)9Rq@3SlqoB7RCxVmF3;w&D%^X~cB-nXZSXmX{8Uv@E3P&@<%6y}@>fok?XFpt7N!%;4qoSGqiquCC7GF56aB@5RZzioU7IP~V6j$1t_q zRZ&+@JE^K##r3QL?!4#ZT?1Mj{>2Mj3+5AjL+b^~$EZqNI}xQ)8E&QAxH0*frUA6t zOI3VJihT)Ki#sc#DWsj!*zRRM)metyDqO(9t}3l$jK9oLUti{LtXIj(a;tvqbG^7h z#0eR`NG3LMm8%-}2W4&tZo9XaVN1LXZ}zFo<;K+|u8wyOHK}e~je6XC4a5J3LmBik zsEpo`B#Sj3d?Sdj>6om-3&uvHeRYwwx9>b|R>OZfY?>K#puw&ilQE}9K zUb|+|5fl$b-PNFa8;eyh&hfY~t5EYi&=>P*nQ4vc&(qRCvDZ_f`h10zq(ort!lOV^ zf*G`>LOwf8W%I3uyj-Gv=E9*MEnSaDDBv0LgXmJ#zsap-oXUJ7$DkLFe2~Z3Aa2VG zL{7pQye_|_D1ZHe`C4wdgX+|}3QS{o7$uPs9_Crc)A{euUtxhT9_*XG^kaPQ=*u$r z^;hJxRj&@q5H`Jb_4}=d|I~KR(Ou8woSkvyO6V)k|9;(lMfZGX&DcLp*tYMflKD@) zQ*l%4x!<3jntQBhde>Js2d;hV>d`wpzdqx|&`)Rm!hhkB7oK=BJ^6d@FUtJO?*>b^ zmu@LNY5T_Hs@ekEtLbybOuXaFkDeKPYJL7$mihfVxBTvvi8VJrHSmv#$N$GmasM%H z>34qd$oW;jin{%UDew4CeObN!+w{X(`&V|XJAil5ImrpgXS{B}n*o z0bCEr2c1g5xqyB^1@NnYt0v(KMS#VChXBg~PXaap{tU1n5$8TY{OF0!11f-nfc#?~ z3*^9iVUhvyUKnd`C*l1u%>=vzI2U;Sf4CTb4Db-@6L9mh33Vsn_qdMpgC7vT2cmqk zNpt5C6$0)6ybG`!Q1AnMjaHg2;l!AY{!8W)EeAXacnjbgfX#sSF3{Rr0RJ6)lNSzM;m*=6vyH2x_la zx2d&wD~i9%(;dD-_cl+B>P?p$T($~cW|yZ|Icj}sx;$ry&$2l;9%b2_5i%{CGo&24 zFPmf74Hfw13m--Mf8>@NQH~TJDL=B~NYjywqsq~Oqv%;oG?p10vK(UY_vimT3;Z8R Ce@5^C literal 0 HcmV?d00001 diff --git a/third-party/mimalloc/bin/minject.exe b/third-party/mimalloc/bin/minject.exe index dba8f80fd2a0ef8b0137d50a8e67996c5b84a9c9..bb445706b65269a336d6e6b3ba2eca4f7399acd4 100644 GIT binary patch delta 10571 zcmeHNdw3I7+Mh`p8hV=mO)2z33Mm!}H4ubCK@vz{0;!8oF3MeKrE4vv_+6&Uw#!&P*D7lMTMf)=buQGn;Oowr1u7d#>nBA8qX^F>72~<3aa-_QP66fv0L& z1)i*pQQ(Q%et-{b`cUEPO&@OcfDS%;p;o8BtgR~iK%wUruUbR4H+3ZseZ8&lUPgPm zn4wIpE@32-1;$8~**;1f$rgmFh^*^E(q#IUt4DrzIEGt3d6FceMw7|nwd z8Lsj;5I$UDwe8JQFrPJdA9d*s`>YPxe0L7SRrS6kKEkt#irgrhUE-~M%&VRxnee{- zl2_=G&C;zzwWwx@!71X?)OxH(_=y*eHR&1ITu*x8Xg*S*+T$YuBV_U+*?ev^nV=T_ z*ieN z#VygP_whoL=lW+Z;SAsKO%%85fU}{~7CrJ!y^x$?3`Yms`JPP zrVCsCSi6FZ`L`&2Gt;X_>72rGZ^V#7V~%*O7TG)pv!aR>FZU%TvuB$;f>JPAeroq#Vw<6>kJ4D4H9Q>C~gs6DXCOl{Mkb|NdCEg;h*zl z=SwPIbei`z}N#VHWaxrFG6)6lFlF%%h z=Ys9XEwNg-%AJ@*b>V4{O8h{cjX`7aQ@9~68h)?%nUF2o`Xo-+KpNCS381s1K>Zv* z-LhFnVkI0XfUY6blmMy|r^E~nv|-EKjFtCg^B<`Bs`otMsAC?XsnfLA2&a_Lk686DWZo#w8 zUDfJ0+UqF_Qx`Y#g45)-*SolXEYi`c-N6fYT8e0};=g0eMyCwL16gh{E;c=J2b_~4 zH)Q!JABlg4h~N$J_E_g{?&$-My#kHuPa&yEngVl#CE`iKML7U|MPh!4cw!g|C% z#Z8+z2(t#WiQKNJA(i!9WiuvZat>Gdt}-#}FB*CI5W>QLlo!r(`(tHu7rNVnXkY{^ z$mbyPy1B}~i$mjw&7O(G#S3j-p`OqFklU3|9fu_GAY0k$zk=EpgO>A4<#8=u$4Reg*64iDf~zP^FipZDq;g{6t) zBAim3)3r-${YMEnUoJ0TKML~`dHqS!r@0XKaw!3{kg}oN ziDa#^^V!FMKcSfPV%@UfnJ2rChn2 ztq-qRmk;yK&?6-*xA86nv0IpnZF<6wm^t3Lhz!7%vPFxWc}KX=G$qg%Sc+=IHj`y} zGRV%4g=vZIY$P?+E3dvSU!P=ebqjYT=IhhF!cKLu^=I>|u5p#mV@ABfdaJ_~iOX>c z=RBbUrtI^s?0>nkFL~a9J$i!NkRX8gn|?{gSZ7%05#rt#@9#G}$7{`%n}sG`ZgA?m z`7p>(lL;mg?+9Cyr$?ejI71C4!aaG%%AXQj`=tk*!U^n@g$N;O$G6BJE^N3+{2z(l zMl^`MnIN9gh(8liO@#LF;R35^-uwj%oeP```3)DV;h9&sf!rqD*{vLjNC~`>Y3L^4 zP%@41ootRBhByu&=&&Wb5y;XosPmv#(56Q~SDl0@e#1ess3g*5OBXkKk&=WTofC6> z{g8;cAr3}v<|B^8iCO+@{Hp6(Ke%1+kvmw~{OVAcpgyYP$DUIrmKVOGRz*l?m6*eG z{Rc+JV;g#f*{0#D^GdMYi@n4%Iq{>A#PHz8rC6RuPVG746i=F|N z*X`rDU3%n#%6k8Wjs*0}S0R5GfOv&N9wbM~8?QhWC#V5>oqxx+BVT#MUce$ZhE?(o zAxh5A`{AMS7%3vbPmBNDWtn(fRU_$#;BlfyK1hho=m{OdKwQ>bW5>&odW2I=mCAe} zS;sG=)_0>dZCKtTT<{7$YnCVblUc%Kq{|PGDL4}&lm$%QNYsCM8rpSSKbLilWJw~w zO02nHpMzBpODI#l#|tMVJ%pSIYMxLN`9U|!EGO5`%{aq;#6J89=kcLNBOf}1`K5+f zq}*P^wUEO++BWm>$H5B$tfV*ZC+_z_Bvf2yI9}*a+rSF<|JV&CB6?{N7x9tA1|=l8svAhhE^e0^H(o&v7kA|~-xwH;kWrqDR$cRrvY)N-3Q+-^ zyYbIJK;s)tGP#k@J74}~XB-4iP>X@!dIdEyaZ_yi^)7+lOBv96>fbjvs< zkJ@w7Lbo%F-P|g3Sx>$>x_ypOBqEREG{*i1m>)4GsmTWaL2=w>Z?nDP|)4 zx2sn}(v#9OuQSp7S6

    !#Oz`N-7x(`eFZ*X9QAx?O{mY3Zx1)>#l_nGS&qz(d`# z&xb*Ni$>r}b>gIyp@Hhb2wbCVUIeK6`vQW+ODp@4bg~0sC!1GmI^%xUdHtr&TdIyj z(=jPJP3UJl`cdqhlmybb`B0Y8f>1o0Vjd!VQMGS;)kgjF@6jPGbOzdFS2Q4bTfvC#|0(WHvrX)n0B1U*n+)d(;pHYz-c|021Z;A5rR+<< zl$4`^bnYuz?wp1~~$FWPlGBmwdPqzbAbp&PcZhQfM0=Lkqg1 zEg+&nA%v5l_Y?j0O?o5ITQ%j0bb>rh1RnKmRJoK%1+fq;G+^0Vq;-T(CH%Cs1-POS zPL?RWSJwL~0@AC|7-J;<698#m6sjb(WeT*h1LrEVln*8@8FX@8A*Jgvs7zo=u~eN0 z553(=yKF^V=>9p_@i7l%v;Y`hIoj_|1*hBe1~_N->ZTCgaHu-uk`%1BrM$%Ve*Z_I8P zvW_#V!Uej4MB-Zjq%McM_Ipo?{mmA>c;jlZJ8{a8Arz+~wkfcQXy zJR}$;3*M`+KBF-sXf)qZdChN-wnD>x;%!zM4tIpc&OyTIdzHzCN?s7}`V5ppP(o6~ zMan_99{ToWTdqy;Q@9S-sB>Kj6tT}jy9V%9g33J4hHYhn%Mrv!)wOuTDCT8k1dRB2 z=>OMP+=vclHEw4+uOrH+4W=DC;IbXlE1q=^Bfe#xV0U(kf$Phr2A>S^J89Kb*cgJPfzm8uOpLGITFk=2vTr+e- zW-P+cegZ)fi^DLQ|AYCFvC3p1LetL#@B}xGXE(lo zegX=wNn;4FSMf-GvPzc`?M_CsF214N!rOt{-CexBf!{+f6Eb3(9|pp{O^mjh2KsGe zc?sTMuD4+dULg|kAm&&H4fBryA!jat$)iWNvyDd&0Kp-AKltjF`&|63b#UrYe1znl zuE#lRSgMF3hJ1pd%-hC@wbr5R39->SG8(jNH$X_|wL>=l4JZ~nKJ*{fXN>TAsX@6lZLzjd zCmRmd11xRqn5smm5`uKtGILn5D`l!5nYrm zd?c;YLY+?e0-&=$b}xN+>?516isyz~liyW^yMHCz4PnX}td`H3JRdK|>N1;f1#acG z(PsnUgmc4{;?hEob3<3Dudu`=9B~O<_SW5$mAGy0*fuP*bHnACg{A)R(yQ*+Vd^uh6`==Fuph%=h}Ko7zd7)ReQ4%6J;&-+%y!iWw=_MvU4n_WL>HznQ(e}f!9 zfi`HNs_(W!;L7V-KLFMOke=-=MMz&pAn!v?MnJXy)NJ?)Bn_|G&<+gyoVMJ1K)JnN zCsw3D>EF$UhMS@tK*MRfRfQBTl#RydiEZN>KC|$#h}QWQG)?6~*BE43Y!A!O#z~D_ zl?2m*A&7=M+CweIX2bnJv9L_oa33(|wFZlCX{g20Y$yPUK!JchDer++8~mOG+%>Ef z4*duZM-i3R>L06gaA48_e+G1tzzxNxpmsp-Y&eh>4;<=0jikwk#&t|aIr*OoHWZckU%9NQXCiH1^*l;_2aNf|GMfZj4$N6FLe??0^ z9KQ_!Yw}<$ANurN(3>@b=8D7sSwu(e11kCg-1ZPe@M4KI7bm3hY8yIe!uRN<_#kbs zpJ-@@oO`DJ7oVr)%o8ybR|l1Pmr}o`)X&Ef|EN-LRO)1f!IZAC)>b*)gFDbR!L9W7|q|}p@`gWyuC^|Z&eg_=w{wTgNVuJ2) zY~2?lyli0eQmiQ+<1!VdYH+*;XK8Sm1|QMjX|3Ix8hlRUAJFJ48r-SzRqOaAO{2^1 zQaAyoO+}a;8r-76N)0}u!4H3B$Z@x7@-+>*%~DKc&HX4UPg-#{aCw=&!-?8Z6S_EDfsNG0ZpGQtZgCm@1c3|?VN#qgCo^expl{L z?qhZKvEpQ#6&FQ(tz2fCrXY1&Y+IrXmIue=jIGPP?Q2$?FkysPJR!C2$q6==9ar~m zE~}$#C+?s0y3VmeuZF5j2Q!fnDBTLwnF>v#g zsDM7G4?YJuvJq1cXaZz$I1EI26(`M4Hiog`v@gfiFNpFf$|&5Be}{t6Fb-_eUE>)B zf6UD31zs``#IFiDzQT1r>;0u*GeJ$>!?W6KJ#Febr^K@(g77q_eDzA-N+`n#{AslH z!L=p6`&WaeK~#$wQwIbV&BHn<8opIlkJAa1S@8?r-OB2%>CG!p^~!$1Q}xz?_YS5h zXPxQ+{ffSm>=zSl#f=VkI`EQJ-U4FUiODR)FYMG%Rm9AIMgUdf(9w7_E%1(EEQp>Q z)jngvFl)fhxu>Fn#LCa~VqeE!GGBu$8N}$vFi)cp&BjDAG2{9&F=P9bWen=W3=)q! z2IQR9G~1yiqY%yF{VZkmNG7FV1d}o@i%DUvOk!96c2k*{_jxiogW@^$H=3qJ!NCEmM-YJtpe*xYK;=w19_ojQdVU1CYac)$Zeud7K zalH?7{a_q3`wIhOD2QQd(6|F7=`}KB&@qOqQ3``Tl^(XmQiD0V7G1#lU2QdKjU83Yr92 z9n0ttZESoL6F)A7iJvPTn>Hk1HZtbbDC43TbN__)xUyL2s1|x)83Ll``51vv=0Mae z>ftcVD@b$6PHePZycY24!LwBf;AZp0Ss~Ip7&# zu)CLD8F<~$qloQ!S|}wtHf0R{{{yvWN|^tX2Wk-w|9}5Lt(!E%7R5d)-ZyJdV6tvz zp>Mjsyu7f~H>Z5bmu)>;VZnqp!ohmXT^--l0w3m zr}duFVqfvflHx}S=Xr}Od_0;ktYNONWctc7pTE3t`dYkxt@AN`iRWFr_C9~v41ejW zQeW}fQg^AZ{9zn;BwA2h?(?rKnP2))aVffdUGJ%I72e~&_uj(tIpu{F@RIq~Fu!!o z%F@*(g{$4`Ruz_!33%cwEN7VQhIxg)|53a{d|-A8`=S_`T_KwEr#E|zsp_k`EK~YH zw!=dQn)2xIe_4Ih4+SEtpOpOGQp<^=Q}kZ1KGcLBvD&*rT;t6W`|8^_-(d{UDYnAI zFfXH21K$p4pkoBM32-e+E%4q)3mq}1P_~0l@MDw~;JptOI$YZNBin*cuoHzrChr2g zVuo$tOcHKoo04!d08tAVPQyQ%0q=cA^gci697)Z$D=trprPVo0A&rw@IIyWfa z6I72275az48PW&rXN7YGxD^tTff6-eg_D6N11DI7k_DV#1h3axDD{P8eRfO zFYjcu8?Xk4PCax8?nEIz!E+kk4mcPeVThj!Xh$KN;{e-HB;bKW8^hd+LV^OoEo0yS z@RtC;Lm|F#EW@lqiCVxgs{v~@oZu%KPVkl6u&&U-C70QZk_8+WUF9sb0jD#SPRre} zN$}!$M$fd*uge%&&DJH%jnfTY^j6th>)xV^Tp|T)ZE~%pHmlZFn_CN|qwIeJl?|C4 delta 10582 zcmeHNdwdi{^6%M4vgE-mTowWZvI!fA1Y*b{2E$_)HqaY45FtF&Adh5$Xb3U8s{z3c zW}{h$^`e6wr{2kVzRsVXIZ;IPFd>ix2+B)EJv8bG69Q*Y+lA2eznpe1n$znsU;ZcWU)p_1>ePhLa>wBNK2TAL9q0J9mw_%$C z@89sO0)rdgR$%pp*8!h1u2W!R#k%!Z&|dW$--f>_FtlEUYnArgvXWIy_oFF^V+!*K zDSa@CG?dONCEBltq>@-&|B)megfS}7I!;XGR*a2XNJu-Q)DY6i0ag;SUdIU9uYr&W zKnn{A5zpl(ac1!_cO#c1_S22lbtH?~x=~z>c&lzSw|?73-MyTFMvQlFGZ-G^bcg4N zn~W=_e1zULlL#L=2!w{qZCUTIacJ0Y%kkjlh|If5wiIWARM+GmL?e{jnF|5AMYhZo z$46OiXka|y1N&Kz&?8$;U(Z_Yk@>llHGiWpkF&nnZsBJtyx*!PvSp9BA!_KrFUK)i z#ta+=N49Jc{}MI!j!6DxDiqr&t=(f?LglpYA=YvAN5jQ76SLIJj7+5A^){=s$hp9| zkZwF@heyom0JT|K*Vobnlc!UE~I4WBxmIAn9(0d8}_l@F{ ziI`}*DL=-;7$0IN|57&S?RBz~cgaFqvt+inm9>w%p*tuvH%q*|xvX7yp*-XkTHP1F z5-!u`(*}O~?&fdJ_RG2^m;DRzkEX<;&8uJtlm)gbmegJFPpp|b&Y!4^ZJeq&3go*fYzoEh5y<8=PKaefQ zG9X@Z&>c3s4>xu9SH?j^%a*wi^+cYwu@H3&w_Dx9UFmMYm*p1j%W(@|JC```a2B`p z#<^o2tv!RlbtY2bliKQX?w(SYK&$iW_61D#Cbl(hZ=u2vr#a8w#J@##D_ptFzByEw zXLSjmurQw%V==j9NbXKA5Q}3*BBMDO?x2zQZ(yc$=(6}ojB`$Y6bzF0&Ym>+ zbl_q^-B~`A1Bx_AmSwgzheQfZ9=6G_^|3zJF;{nXu|;gF1)m%O zRU8p()tyWi9kFS;pN-0)#2Xl}gN6+3-a88kp`0>9mm9BSf29axXanS5xE zvM%e-n5ev$Vk31>;d6dhtZZ?h!`ld3!rWm;Bli*?+9_7WjlB7HqnUFKd4wjK@e#kh ze|=mwjoib9_65E}rjlJbYdIQmbAbzj(To|pg`P0SZ#SBN*EK%GED%1Rk;yewXr_lH8o5=b&8M)B@~q!%v0al&TXR@mYZLbjIxy9m7L++hGpBbF@?qrw5%G7sxl2Om86GrGtv zw{+=%2=n8d!t@-s{zIC3h_Cw!=Z*cyoJd?c)ql>6@#qf-ixb>JGmMxSu?_QN?1w@6 zeV*_pWh~fOAY2BsAmc}IYyUyP)oE-|2IUERu(N2`Wy_Khh7wXB%rprHz$nN7qu7?^ z)*qw#15~f7k$Q%(ZtqJ`KPrLf&!htOw=h3}>W_LNhA_@^g z%)$&Czq4nED?G!*@060^!SD{%i@%+@6%LFoQen4XZA zaoD3jRG=SZM_AmRP@o^-5#CT{T%bQuPI+0hIMWs?ql(UdDO%rTaMh;tA@(Eg;uBjQf(`aS8&|5 zyAx)IZHdz(hFE89vzg)U+>fdr62}i15iAh)Vh6o8isgdC(%|orTwK_Tk@zzpNzts0 zMl$>_UHMLJ=vd<{YxMoGvCQ-4FIcFYr{$i?gSjpmGI<-8>BI75I@C{|2Tn!)5u*a6g z1bU;(BV^IGy%H=chpe<(Lud?p+jfwL0ZCSNL7q4#*WmHgK~A!>NMYZ|)uC z7LLJtvJDQ)J}1kd74UgK6$FKEU4km!^#uC3g%4Xp%EXauUNM1h&kDz)w zFLc2_30iMpS_#vN`fGyyvtNAf5ezsq6+Wsh>BP4Xky~i)=T+Zy$AK}Hb!-FGBOtI) zX14w+gduTTZVE*5Afjv_$Gim#Q7U9n)PpFdBJrOw_Jd&WL`;!L{%G0KIRYvCGt{GV z#3qX+csQBO(PLXg>-O=X$%s|^5f+_=uE-1>O88#3Yng3Dm=?Np7p1#=q+v7nPvC~_z&)VovhYU?qh4k&cxK<6T{e4_Oc2CnN>J0TMW$R$jx%AA%oPc<$$X-)*Dtr!bk!) zYYFQr&wfNnY}*Zk>kR{FWG;t6&(%zMmt7@sC#^+Ceqj@hObG*{ks_AW4^Teu5&}Brg@dxi%j6m22cjCZ&dNw1OF9@-Gv(ErAj*dxz`JJ3pI9R>RV~(-oOGf~ zwfY-O@EWTJLgQ7d`_ag>_sr$(BSw)!x|%7!UL8$C3zZI%JDYlQ|D9s?2RyW!9)NM3XtB$h2ad zlf5!WrJI?|)nA`0yZF>FOLEDV9^tf;|6`k7&a+>@1-$m(9RJ8AY~5XWi1@*<5iuV? z8#~s$@#4i{R^6M);*g}Isa-jQa9#I8swQ__;JTXoxV3?SHTUUAAgzX=f}{Z5P;VGka_){-7U4^}uuv3f0cMXZ6JdVPGGt)}FP!E2=vZf}=XF?istB!w00Tph&YYIP+F*4)Kpb9+bZ0NRZ8IAdW_7 zq)>+l6ZEnrfhil=M;OISyVMGkAPYaoO2_c!LHa#js9U8s6__Z!2zj4w;m?N}w4-t;9l#*4E+!|DI1VTOtAt~ZAP%Jafj-QXQIU)ttr7Rc4V%9WwTEY z=`D?1_r+hL3>WHJFzfayA7OY`OTE#Dw)96BC;c57k?RfFP2CAnquS_a-zPl}wEOhK za4-9k@r$mSDLLp$giiu1;n;icl#H+uj!R#F6Uv5YOJ$mWs}KXMd>g~sK~*RdgVrF(+79`4+2h5-edHAlG zXO=CS(87gkJi{whJW^m)+g8@jv!a`sHg{U79k@NuOzq9|ZFXsp5$kj$8un+!w?>)= z$7eCk?8EBCLs)_f>`N_HU;KRJkdZ$yo}3m1l^q?8TuaApAUK%Mg05~sQ5XLa!P>Mpv&2GMD))fsv!%ukVH+S)fbVG*ajR_zrx$nH9J>Ae#edtf zbPwpo%rtAs{r%Yyg=&En{wS#(G?LfDZ*EcUaO@yo4FPtBJ7mjkK#?%w;0aTN>ZM&u zG4-_V5PU>8?r8#8+|tpNt3<65woE~>_-fjG-AD1FVbsiNMF=<=#)qmUCTX^cOCJFu z%2}DjYG^Fw@L&oXbV@Zq_dx7uRF3{A*|Jp(kFq7sQ<Jo{%Z;fQl%PlexG?%eoYS%ue)PtrnI&N_-ZGo^-Pm_+GboM($SNGb>nc|u5IUBVdFZW-O(<=csGz8 z*ai3uK!42bS&svce-=tz!e>lruS$OB*Zq7kd9lstI~85pjGYKUj`a)2p8(45Je9!L zKB^MhL13Aegoih)ykWkzd~oNfB%crZxkI%1D=UpfFe&vK?rNzmH?;&73Pus)we6-h z;{!mDY)IHx1I&4$#p=(gZMU`=%g}_^z*q@v2c!*=FVapz^L(-gm|^`qRpL zf%29V`cMTwqu~Ei-aX3uJLP>|c|W4aTmn8<*g22Qs^dopg@8)H1&WBNnEJMX~C;Mb|a^BU~GM5U){aH0n1 zYOqn03u&-Yqd%#&_iJ#IMpw0A`L#B^O`!zIODaMh)}UX5#TuNe!M|RkDDiexWsL^! z(O~aDom#*B8a-}tFGqzJiWv-EC0@8jflU!0WTpmJY0$612Q(O?!7UnmL4$8-aF+(# zHF#1RXAd)wjlWAHjMXfp8Z6adl?I>Hka@>*zK2c&n<)O1-166DQnwUBBMuIC1ffNv7V`w(1+lbDVwK%Q>7*Y`p0uUHVc{ zzG;wHncJWvwU+2c{-<0(NYxeCwgc(5oW3w!+} z>eY#mpF7Vr@780JCbw+5?0ot7YuxjlPd>x)?2Ma==li@>zT(nl%PNXXz4sKaE?d2< zynIc`xUz~ny(RwQH6;Om@oL{aCF80mPAH!1E%z?-c_F26Gb>A%LkoUMc!nvM3TMqI z@-Ryq(+J7*d#nAn)fJ50s2S8tvt?z$$ZbDQ9>@(KICC?rN|*VUK@bzAV#?It1lgiq z!Xl|0+q!k@N|&#MfTM^Il#!tOOKfhZ6DXm-qTQwhUSVHakt(<7S2R^_DQK^-ajI1d z*k8imAf$q1mN9}2WgONO#6&=i#wtg09JET-QI(s(Xe_UVaN1}z<^n>N64C>Zsm-tl z?FQCf?(+ete`*&~ru{kC08>&JVE`eU@vt@*V<0iORK#RQRi-3Fk%WJm2wxgaj73qT z6EZTMCaquJHkT4lQv4YtrF0BQDY}lNOh_jwoQ(|aN$ecd1ieu@5|ypX;)cYKArl6Y zA&*(aX^#HE0nuc@?)aYA&VH4qrO|~^Sun|{BgSKKBx?jAMac-p`JkC}#Iz?tY2OW+ zhtXg<F--lvj;N}1VDLV)`;q1oorG+;wO5Let)Ml5X3{b* z<1~SmH60tUPv5T=BVdk0VM8CH4oCJ5&^r6j(m^vp;h{b>3fd*evGDKxZDmJ-+G+uPRwph2wGCeR+co!MTQ80^X{%h2E+aYpPZ+t0?i#SQEh2 zsEVZMW_kTH0##Ms3V&hMni8+i=Po5_eW*pNs=UieK_uTZ>b$i+zjt*}*=n!TH?yqV z%P_J=@2)8Gmn|zVTj!nUDf9U$x{wbIbN%HrmR0)W163Fh<5l~~KMdlC{KV_NX1oPu zRsO)T^7$3_mQ_H1o!;$pd6x%PuJl$FR(X9ei8P5z@`o9N#(7@9xHNwVw@Lhc{xEK% z_-4NEF^@?!dlFp-7h8|4I3~-z*Ls$X`u=iWU3>Sdum7lv-SpNI6+N#_pO8>AbVJFi zy8F9di#`x_VYRv7n1ecl#c^*$C@J2u*PL}n+&46q0 z)C2E(GO-it7@jSlGx#ZMCcxIwD1aby2=MQyymElE3hW*nDr_>51VYXLM=4Ks;~_X4P|hpIs0lbbPT5hp88Qqe;dvLh_S=?UByijM&yCX! kUG-YcYxS>@LbZA}Z%yBtwKZp}V=LXd%~|vwF0JwZ3xZd3e*gdg diff --git a/third-party/mimalloc/bin/minject32.exe b/third-party/mimalloc/bin/minject32.exe index f837383b988362becd9b279b5ac254b7072d5777..6dcb8da9ccb189f0e31daf31b06aa9449c840f9d 100644 GIT binary patch delta 6814 zcmd^De_WJR*1ymEfKeD{Qb<%3lplilM_?hHz4wn@4ksZKWazqXqb%e9aYJ= zesr3Qw~gKmylYqOXgIKXtoEA%;Qc$c$a4F4JoOEg-@l_y?%!T}my7CtY+=mfjAqgA z&eq&Ru786>!9*}KmXJg+l0(ey#o`jCR9nBEkdu_NgOEY-_vulQ&4jco2$?7(B=fO_ z^O#Jrm$^kzpCBp}$%>C+#3hPk=76|faU0X#aagg1VVXM>s&IyJi__Hy7q^bNuLM1# zLC<7qFgiZRL3}47TRYuhLdj(QP{(GWd(%eYpH+d5H`QH=FsGC&G);;RYBt67m+{U? z^8#HPAU=XEXFVo)>3%R8m@!RUI2gYeF?z7r~AXODA&+TPpnwau^m?xNZwnsGVfP8xfKRT{oZ5?`L4vXOTV3cKU|yMoF_J&YH+$H#@08s*8f%e-?y=;A&7 z#&hH2X{Rh_bgdOwZ)i@b>nwFb8FB*V3soau+!BOKg@eM%NM*OMCezZTYc-375oxKX z_`-p-388S(9~&m*sFXcLbxbhYg?$!I3eL%5zVKp@T`yglC;ld4zW3}rcvLuvypsNi z*b5#>INN&8oeVGikf z)!c1at!T_Qp1ZpZW(uqILf%cW%2Q%xRL_)&=*XKG z$%jdn@rEyjs!^AlGM|^lRhu^n0l7$(xz3`1N$-JO6pTKGe1vgDWv+J0UGfIlWv&rQ zzYNlqk_+l6r4CT)my{}}RGVnKDRo7ZH2OUp>=!;1{=`=%#z-F6FO+D6KOM+R)Jfr! z!60^JdL}xQAwGFixgw8UpO+9-<-Wn*D@arU(*E9wQ4{K-Ppe&?Dxbg z%z={NP)fI&+Qkgb=~mM%WrpNJu- zyf8a<)`2Y!;`fhS3kJJeX-lMEiTaqu3#1?TQr%ZnMLg~S>p`V;nLDnfQ!I)xO>cfn zo}gUL12uGOnRu9C?_ zN?Pf-Q~-(tb-J{Ukv@(jL^7&z8YHC_q4*9IIK*J^rlE=7oaa}&LR2r@+-4j5a}A@Ixt4 zmV?x+Qt1*FrBa$;hV(dHKq^HmOh}CsvC`c@Wu-w@`XqirJ`V`K8V+O1y)EZGYUy?e z)KS06#HZ%XkNtEybW{X{8cFKLG7#gMn?5HHoE;&aofj9qS4l{zbO>TW&!BV)__Xw_ zsESR#WhffAi@7keVLEONWtZ6**$|2A!SbQ!b7SYGs(|qB(A83DSR?OUeD&cN#Nh3y zt>Rs=OT00N2_ro~x33X?qe!T@&Z>ZPcc?SHjz}A6Jchn#XQ~1v!f>-~4yF*diuOwd zj|OitweK3^xl^hMt&rS0=2_&5g_JaxI)$y#xl(91BGh%Kah*e~uQ$kR?Np4{#>;J@ z+)_ziyHsvjwB2gDAk)sklRCA;!kW`{8dnj`j3V*Y`8mu3qJMs&w;EZ3+|)HUX^3Z< zvGa{6Onc~t(bP}k4r@kcd1eMZ8cBiuqxf3{qoU%vb3PljCkVbLT>1>|3`K+-=>uI+ zVeCaE$Sc$cU2-Vs^d8ix@&1Z=owLXLSIb=AaFpLUsw?X(x?4BPIV#?_z{WVlmll|_ z99tY@{Qb?mwi(2x0CL{g=?EMnuq@!pqzgyu9S#y`X96xllmrcf6p<#v#rU|J!yS+a zNZ*ULxD>};6s*9p-L%D?>>Ey0B01`22cwrl0ivK53N?RDty4;e;AW31@w5U(vULF0 zKpl?tr(xoAaR(F?Vd9GTII}R~cfNokd1D+;Hy#ieh@W4qprUoYk_ObSn- zbPA;xE&uVtmnqkCs6f@hbgV&yyhJ@;pQ!f((gur5%MLrlcNVhV45$uzqWG|&M<4V= zmb&x?9C#XBsTC*3cOn*<0&NJ4FR6$c;2TcyX=5P3)J?w;B3+9bF@siPr@yJ+*juc7 zR=BdUjSuH{srcJ8#$LWG5|16BX#nxqH@>Q7y9e)`^xEy|HEZN&)L{B{cTGLsPuRrk zll^#X;W;ElIEN#8_(m0ftA;O*^q8SXJi2JPH{7S)0DBmrWYBniVb!oPxUfp{HBE-| zC8N5BAA-+y;pIP=B5V6Wi%^BP*G0Z+0xOOOXdQE3SEukb^)q`D<*7MSE%|XS-h7zo zbxL)KF|>%!julqMQoOF5+JOrqDd6o5AN>KOW;f^iHX z5u77J;izvgEa8lAB23qs1_AnCKmxFApzow6v<`MMWYJh4-hIEImCZdFFRyzXN9pWg zy0kapBj4cnp&7^_azwUA7PfrkS%>A04}y&~6W&cYV|mxZTF!Wu%2Pw>$->ErLPy2G==~2G`M;QDa#^7dhq#E5GbvfE;D$?1bu9SyS zuGFsURGq5)-cHifT#r6YrBGkc(zDJNs3*|x8bq4XdP@5)R}PB6AYNo`;-aK>o1Cnu zgnKLZU42zSa9QuW`WmpOK=*PL{(YJBjo4q<=gxgm1?`^jV~-GmaJ@*c2KO>?M)JZ1 z#~uYsmv>U=>J1#jLl(oqB$ETBI;qIv5W5sV^X@GtjQCSSs`!#2w&R?^#4x6ga3iBo zLZ29x{)n39h(r8E`etR`I>%!RqeXp2WYUeFt?&pY_1N?Fb$|?X@ZA`jfnAOBKDUgJ z7;u%%%H~Q)Swa6CnDQc)6S4~MHsD)8(h6h%;7!2WfG)sNmJk!*5a1{voWzL?Tw+a(KM_bx{57E`n4d23T0WG}!4y*Iu-=rcJ8`JXAg zcBJY7@S-qw<2wi5Ab15+c$dM8L5L1bnHz<3>Vn+EQ+Ouu${oZ)UY)|c1I%U!{&os) z4|u!5qmJMBj?n{+6ukNWd#S)N_`j8kj$CV&irFnTIp%mX6sv07D?JSjHQU`q4fowu z)7ZGCnq*DkZg4f!Y^?^9XqAvHzQ563vu#7|wwj#A{Mz~&N)bW1W_zu>c58j@o|@u< z+D7+kOdp^Oyi$BSk#dM0TIX@N8Dh40V)obuN zkU#u7o}ya!nd42AAFKNJbKc2gI-I#Y(=j7&jPagE3NdUdyNUI(zhqxy-(vs5{)?Sv zh%p!qm4?HH3x-H;E|8EPJf`t&dxOYwfc3TQ6F_vR<_+ZCbBw zwr!v7sO=5gc^k>rW=Ch+vWv1SvTL*VWIvkybas38>)9u=Kh6F&TV=n=9%o-_H{0{= zW%hdeL-wE9pRvDe@3H^Ie$jr}PU5L2)R{=On5|}i&7Nb!4B3VT!}ErZ4KulX?g8!< z?jlEw3C3DugYl=v7UN-~x6}A{<2OdNX|~B`+HHE#^p0u3q)d-VzcsxieM|bi@bmTb z!Sq!b4H*w*Je=`Z#w!`GXPn5mmXVNoTjqnAFJ!)+c_y@>e*K5qWRJZd&&t;{OOs>r%8>&2{BvbwWQXI;&Tv!qy-TXMV>tu@)oTGw0K ztWR36SSPKFEy|W+TV`|Ew%Q)Bb=v-D8?b$5yJUL;QLxx^>}&1C_DcJmc9*@$&f6bF z6wbr{f7vJO1jm3DPr^=TquB*)BAdsqWjC;Quy?Vw>`&O8>_PTXHo(5m{*CzvEux z-sHNt9`4WF2i)Jde{lcgE^&7nT?Qj@pv1#y6AUOY6d7EGdV|}x+tzIBw~g2^P=pE& QlxJ9NsK#i=`s5?ZNz;7vBc&jPLg^Q_K%v33mI7^}fYd~j zIvfshz=K@JQJ+D)>L4mw9l3G`TdFN6qZJvfL$6bPP>+_GiCSs6R&(E-l#!W#^ZTAY zYwxw+z4qE`t-ba-r|}(Te21BXK3C%Yj2jh0`Nwb4M&X0wRTzKu9|g~sNBd>ZJEDEj z^XsF1-t(2{^L+)+uR~ukRB(JI`iG9Jh~f_&`NMHK|IiV8biB43`abp6_KRW zJUj8frEKqX$MNN(MMJM|Zw&6%plif}@$0fr>LuaoeOpW)yCF9CN zoI|9-Y0$u@Jjj0+T*atn780^Qt|-vyxxw4gzv*>crSk0&cV7Zt~8RA5%o~UhZ`6FV_*)e0@YX< z+wvp+-QXH#>p`J&yv7@Xk^JTK52=Fv@xeQoeJiI%0_Qz2GqOLA2`*<~V_@ z+RZsah%4VVZ09`#h>F?Kue$w#D?ddd?H5&nA8@+7;~M`C-!^KOp6&Wn#+QfuXQli` zbx`KxYQ?TW3FjD=p7jro@*jNgwX_fVq+MKjKLTMm&o5I5k4{MQ^1D)l_e7+v>fq`K z-!n487bv9X(!b>#gM1g~88BQN8A*S~{GO_PC-xgMmbos$Au)6#>IXddDmxO1G|F7m z;al@X75EZ{7Jao9Ei#8MP)d9F6-s{jQgfH8T@PhJS+?P#FeiS_{BZg^TuEOrC60HD z@^+cDx3rOoB)O0uSXl;hBl=SD*i>PA{B&<%D%>s^MNW#3`~cM+k?(82=$?mA1eHGC z5$^g_vSux1i!?#mKB?oX)G<&hmS7pyP)ka#@?Gf?CSS?qE0}yWvq#2FfMr2V9G|b` zJ$=$1uKX&Ox0FW+N=CVSHJ`7Y3nzc%S`-Pha7;eKbuM4a=P3;5x0;J35Nfp>F7nIO z(s%g06d^-7+gmbP0fxyFjqUk?P%hvT;eOy zoESBn=Nx@VRSx#qHByPR7e{U?39B9yuqVzINg(}@Z|hGF@wI)@Fz@L1cdHjTLY$|M z>MI}Q9RskvEW{OfM|npd6iAU}yk;N&PB4$*9R2&5JeptWCH+#5xL*=n%j}B_-c1ws zTWIiiYj{Vfh%XX&af7}y zS`#WFo6^vE*OTv8SvM_Gq zbmo*WcVh1Jw!4xb58|jPr!9j8eAr;EESJW7Vev9V`d6xCK_f8eGg4-VO&PT zb(Dc6SQE)SS>Z}4#D4DyD%Dh`Ls+OvWA=BHsb(?WVwm8wWY*gWYEIZQ-e)mV-fhS2 zsdN?6Rj+*)i6Eh)`cnAXa3r&%!sVhfzrjoqOfrwOe9hMmyWA9fFe)zFxPkAE`d;bk zLnRSk0*tD13dsxON-JF_DZ232fGb@g%0C3wm6i`KK)HRC+e5i3%Iy%Q-?o5F7KeX? zgO|BNt%g4*R>FF|Tw%DvuheQ#-X7@|WuuV@4oLGHx@)PY6d7vnOdPB0FTbcx)tT zkY2r&TCFgYn7$zE8yXj-&k|b~I7pBQ7OI)>Zf&xsxWW~1K+8nIHc8K15n3kYF~1gG zoitDON(krPHA$12LFXUADDn|ie=|50Wv&4_b8knYMj~Za2}zS@uQJlouu8#|gocWI z)vp(#CbdZf&dx<^ipV}{1{$TNlZ z$+M?9Zrp;Gm<)0T*BLTK{*DbM`_7gzj^ zsz`0W;x_rriv(kGrgkXV_3YcT(p;%OOMU;>XbSoM5Y?gz91~iTHz{;uGrl>`Al3fju4k?5mc;{L3{)aDkP+< ztJ^z0Ys3&m^PmK;{dm{+3n93G&WhuJi4xijP?6Kog*DNsbKs$0F2-Xh9;MUzII6+J zbT^I@MyNyh#E}0AZ8mTml7M>u|$d#q^pTnGo~GeOs?%6o(o)6+z+( zC#I^|#^_Y9oUTA<$dG*h$Jip`84%yZxLRBW%lMEXVzRoYRfv;?Yg4CG%#C6|CRK>v z#LROmB3Yt|DiePkj}>8YDh50&CZMaVh_GT$m>#hbhoM9O5TCmOS3Gj@2BaIQpVh)0 z)25g1$FRD(n|F%hAF&I>^uD$)2?WnzX3YHP>nK#uCBR@(jZ8jIow31$Mk21Kp&I(?Bvad6&R_Fc zKE@K_meFwq?@_d0cgy`hF`mt0d~An#?bkiCT?3F3-&9~JPEF+!>F{K1=uX#n4zd38 z5iY?-#khpQXnQrS>)LeRCX&Xe8+BbI7+C{EPnLDC)na@jHh}5bri;n3M|o85Tw( z?3iJy)Ijyf2OGJBWH1}LaXk#3_U=;zhIPBbbl)Vk-A=ksGTkmhqy!Cv>=B=YW5}(W zH_QfuFPtUh6sd!-*!l3xG<$A6*i%bsizB-Knikno);Jm8xlo8hm-8ixKQ3sLici7o zUYYiS1huz)0I!xtR0K{gxMuF#6rqXlkYTG{$8$-9*n1r(AddIwtC1uP6g84yRb=E! z(D58fb8Pw4EBHU)r|w8n3g&~3Fy4sC(JehnTmS`je1XOnIsBDVSdm)9d?Fl5U6Cfi zRvgRUBNKmurKns2fD8V)3b;hN%PP*uaJr{uYlS7VW>3ISNry^0Ts+hDISN{R#0b5F zFVw2JCapRMN*^q%s646>yt7#E$Iu$_Xt=nDM;-AfD_p}%P%h{Jh-sMezm2gw2wP?^_TI)@)Ue6_KEp+sp|}`;FSW*TRb0ua$uYc_HJrayT}%!Ah%Hm{wSyzw zwA#5R`rns~>5%fZp`+A6|2avoocc@;_83J*x(HDdRe*kc`p?O{v1zRTd73EPyQvsa zyLTGSs~G%)aSi{!(SJ%xLXB}7=Jc3*Je6bm6)^>fa!kSX?o_wge=ao!qt3+OQ^@eX zIsN8-&kX2!V-Pr$%{e>YeAbf?I}AH#q?V!r8amJ8u9)O0uvhg+XSX>)BTSmR%A35I zmI@7Rlhm}GuB5F|L0gLq?bZWHY_v=j(c981cR`fF-%(6%Ajh;A_R5pdj+c?nUU^D%8h1>dHpVr|s_yC} zy}k^LX;folK7l49w^AO)xNDH`(V{@dE>|A1bPz8x!8Y$?ZuG47ZoR)v@v4M$y2pcg z4UA`<>cv_l*r-2bd+vU>?RPTh_KZ9A5Fv=w?<6>0_d+3K{;Wx!iKl)Q1y3D}cDzF& zY?%L4N1r5x@zzuma-@oo6QHx80noRgIN-BE0niT6USM|g^mzvTW9tYx2x9;g`92)Y||8FM1&G4RiUeh=yd zy+eE8PlI}(IDqjADBcQkg94z%Ao@H8nNy${7$1P#dysz-V>{^Dx-vqB0cbJU0aAnF zK=fIU{xat3(Px1Uf-*q#(Wl#^J#Cuj{mh@9i=-0PESMZk2v^6p1@jq+K`$I$n9C61 z{e_keb$X+$!@J}wCOI{&-nq5T)8tNVXic5J{cdMVOGCYL{sO_JTQL6SaFK9Gm!h~i z>Nue@GK`_)Gd&{_#LNS7nnInzZsSJjTdVAcW+e$Bqq6qq4@orFHiA-DgR;TkxjCl< zuL1cm=lhpA`q|Nh#fLF>_+REG3SE+A-p?1|=O#!=d=lRPes;IS_FlIgHJ==5AdKF52N>a$6aAi#EE>DscwvP-|2e9rTJ*{Y|%_?RR= zYp$Z+6y(&Dw>n!kw${|w)iu}DJMXU9*08OvscHL`MGei3&Moem?H+f{w${71EUIj1 zHt1{CI-8tztxkwVrw#hWJL)$>6P{GiOv>P){j!2}D^UkY>Fka-vJx3@72>eCrM}Kx zhh&43ct4GewXM5iFnJ4j&CmGV+ne0m>M^DvAB4rR_1Zmq_SA2_6F9+-cQHNgE!yRI zc@&cU6SjUm#T09a_h-R%F+t_6g6m>}YGAoousCXwmIjZ|UkG2NSOc|EtEU)2qSNyp z!weMC0Lw&~7{f9tMz3%SO7!A}l<49t3X)BP^r2fk=5^mD;ftk30&CL9pPi4_vasAV z&HL;EJ9z_C46QexSD`J_PRQf8UKe1$zb9Ag~O`-+V3tOGbe9-Xb>)%mulo zTQE%(Aysx_CWmhUa{&4vc;FUnJ+NKCsM|N6M`^Xf#rehm_d3Y_yLGT*vpGk`tQMZI zP4P~XEOWXGJS{EGW_M}J_ASoV*5&nN<}K(tSBtZ*9w50!(X!pGZs)dj4cnY~twjw@ zPRfz3(&fz!?uNRihCR-*l?|=#Vk{vKNSC)3IyZanywllI+Tv`*7UX4ld9$moxxUF+ z@7T4)nX$v&u)P_2+|Cw4cFN0~?thxUb($0E?9-C&rACx&ulF=Li=%pwS5t*U_DRe^ z;Xm#7KAf*M;$DzdU)WE-Jmt+o$}fFk_s$ny`9TtY;PoTTp_lSAHS4CfZE>~VH}ulD zcjK;aQ7R zPjyQDRQ*zYjeeKD%kZv2k!jBC$xJogWqiiiVf>3RK1-7|J!@R{jO@AD4`uH*wVP(; zm~;Abl;#EILi0BBUh{tQqvjLlH_hLgWx2_@kL5m-`%3PG+)s1ExiZT{uVsp5j%AT0 z+wzpyX#t#yZWx0SO#W_`(e+S+IR$a>W(wN15YZ91FHw$fH@+iJVV zcEI+m?KNAc?LFI?}5eZDj-O8TMOto^FHgVO@{zC!JouRsW>^ zqF!c5H53>)gZE*>ZwyZxx(tJc9}P*Fb2GCut1=H{zL5Dz=5S`R(Pms{{H5`b@p0p` z#+QxX8B?kew>?Su~`Z&m6p3KJ1l<7QOg+%vC6GetaGdx)<0Q4v3_BlZOgD3ZB|>Et;WXN z4%?ovJ#Tx}cG>nnwrjTQHiF9%KtRSL64O~No6S1d+u1d2C0omGVH??<>@FPmgKSNZ zJV%nq_4mgv;FDY|*O44q!LRAQ?AVbrr~o%{rH^S?AX6 z(fM=_>JA{VkLsS(J*_*gJE{Adcjw$NjYqJ)bd+P*rK+=oKpG~ Wk53tWn!Zq9tnYZacs0$%x&H?@v_~@l diff --git a/third-party/mimalloc/bin/readme.md b/third-party/mimalloc/bin/readme.md index 9b121bda59..bc115ce160 100644 --- a/third-party/mimalloc/bin/readme.md +++ b/third-party/mimalloc/bin/readme.md @@ -1,27 +1,30 @@ # Windows Override Dynamically overriding on mimalloc on Windows -is robust and has the particular advantage to be able to redirect all malloc/free calls that go through -the (dynamic) C runtime allocator, including those from other DLL's or libraries. -As it intercepts all allocation calls on a low level, it can be used reliably +is robust and has the particular advantage to be able to redirect all malloc/free calls +that go through the (dynamic) C runtime allocator, including those from other DLL's or +libraries. As it intercepts all allocation calls on a low level, it can be used reliably on large programs that include other 3rd party components. -There are four requirements to make the overriding work robustly: +There are four requirements to make the overriding work well: 1. Use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -2. Link your program explicitly with `mimalloc-override.dll` library. - To ensure the `mimalloc-override.dll` is loaded at run-time it is easiest to insert some - call to the mimalloc API in the `main` function, like `mi_version()` - (or use the `/INCLUDE:mi_version` switch on the linker). See the `mimalloc-override-test` project - for an example on how to use this. +2. Link your program explicitly with the `mimalloc.lib` export library for + the `mimalloc.dll` -- which contains all mimalloc functionality. + To ensure the `mimalloc.dll` is actually loaded at run-time it is easiest + to insert some call to the mimalloc API in the `main` function, like `mi_version()` + (or use the `/include:mi_version` switch on the linker, or + similarly, `#pragma comment(linker, "/include:mi_version")` in some source file). + See the `mimalloc-test-override` project for an example on how to use this. -3. The `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be put - in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency of that DLL). - The redirection DLL ensures that all calls to the C runtime malloc API get redirected to - mimalloc functions (which reside in `mimalloc-override.dll`). +3. The `mimalloc-redirect.dll` must be put in the same folder as the main + `mimalloc.dll` at runtime (as it is a dependency of that DLL). + The redirection DLL ensures that all calls to the C runtime malloc API get + redirected to mimalloc functions (which reside in `mimalloc.dll`). -4. Ensure the `mimalloc-override.dll` comes as early as possible in the import +4. Ensure the `mimalloc.dll` comes as early as possible in the import list of the final executable (so it can intercept all potential allocations). + You can use `minject -l ` to check this if needed. For best performance on Windows with C++, it is also recommended to also override the `new`/`delete` operations (by including @@ -29,18 +32,43 @@ is also recommended to also override the `new`/`delete` operations (by including a single(!) source file in your project). The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic -overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. +overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully +redirected. -## Minject +### Other Platforms -We cannot always re-link an executable with `mimalloc-override.dll`, and similarly, we cannot always -ensure the the DLL comes first in the import table of the final executable. +You always link with `mimalloc.dll` but for different platforms you may +need a specific redirection DLL: + +- __x64__: `mimalloc-redirect.dll`. +- __x86__: `mimalloc-redirect32.dll`. Use for older 32-bit Windows programs. +- __arm64__: `mimalloc-redirect-arm64.dll`. Use for native Windows arm64 programs. +- __arm64ec__: `mimalloc-redirect-arm64ec.dll`. The [arm64ec] ABI is "emulation compatible" + mode on Windows arm64. Unfortunately we cannot run x64 code emulated on Windows arm64 with + the x64 mimalloc override directly (since the C runtime always uses `arm64ec`). Instead: + 1. Build the program as normal for x64 and link as normal with the x64 + `mimalloc.lib` export library. + 2. Now separately build `mimalloc.dll` in `arm64ec` mode and _overwrite_ your + previous (x64) `mimalloc.dll` -- the loader can handle the mix of arm64ec + and x64 code. Now use `mimalloc-redirect-arm64ec.dll` to match your new + arm64ec `mimalloc.dll`. The main program stays as is and can be fully x64 + or contain more arm64ec modules. At runtime, the arm64ec `mimalloc.dll` will + run with native arm64 instructions while the rest of the program runs emulated x64. + +[arm64ec]: https://learn.microsoft.com/en-us/windows/arm/arm64ec + + +### Minject + +We cannot always re-link an executable with `mimalloc.dll`, and similarly, we +cannot always ensure that the DLL comes first in the import table of the final executable. In many cases though we can patch existing executables without any recompilation -if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the `mimalloc-override.dll` -into the import table (and put `mimalloc-redirect.dll` in the same folder) -Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388). +if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the +`mimalloc.dll` into the import table (and put `mimalloc-redirect.dll` in the same +directory) Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388). -The `minject` program can also do this from the command line, use `minject --help` for options: +The `minject` program can also do this from the command line +Use `minject --help` for options: ``` > minject --help @@ -58,8 +86,8 @@ options: -l --list only list imported modules -i --inplace update the exe in-place (make sure there is a backup!) -f --force always overwrite without prompting - --postfix=

    use

    as a postfix to the mimalloc dll (default is 'override') - e.g. use --postfix=override-debug to link with mimalloc-override-debug.dll + --postfix=

    use

    as a postfix to the mimalloc dll. + e.g. use --postfix=debug to link with mimalloc-debug.dll notes: Without '--inplace' an injected is generated with the same name ending in '-mi'. @@ -69,3 +97,6 @@ examples: > minject --list myprogram.exe > minject --force --inplace myprogram.exe ``` + +For x86 32-bit binaries, use `minject32`, and for arm64 binaries use `minject-arm64`. + diff --git a/third-party/mimalloc/cmake/mimalloc-config-version.cmake b/third-party/mimalloc/cmake/mimalloc-config-version.cmake index 81fd3c9da7..f3ed36ab5b 100644 --- a/third-party/mimalloc/cmake/mimalloc-config-version.cmake +++ b/third-party/mimalloc/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 2) set(mi_version_minor 1) -set(mi_version_patch 7) +set(mi_version_patch 9) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/third-party/mimalloc/doc/doxyfile b/third-party/mimalloc/doc/doxyfile index d03a70f57c..53f874cfb2 100644 --- a/third-party/mimalloc/doc/doxyfile +++ b/third-party/mimalloc/doc/doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.9.1 +# Doxyfile 1.11.0 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -12,6 +12,16 @@ # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables or CMake type +# replacement variables: +# doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options @@ -53,6 +63,12 @@ PROJECT_BRIEF = PROJECT_LOGO = mimalloc-logo.svg +# With the PROJECT_ICON tag one can specify an icon that is included in the tabs +# when the HTML document is shown. Doxygen will copy the logo to the output +# directory. + +PROJECT_ICON = + # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is # entered, it will be relative to the location where doxygen was started. If @@ -60,16 +76,28 @@ PROJECT_LOGO = mimalloc-logo.svg OUTPUT_DIRECTORY = .. -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# number of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode @@ -81,26 +109,18 @@ ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -258,16 +278,16 @@ TAB_SIZE = 2 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -312,8 +332,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -344,11 +364,22 @@ MARKDOWN_SUPPORT = YES # to that level are automatically included in the table of contents, even if # they do not have an id attribute. # Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 5. +# Minimum value: 0, maximum value: 99, default value: 6. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. TOC_INCLUDE_HEADINGS = 0 +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0 and GITHUB use the lower case version of title +# with any whitespace replaced by '-' and punctuation characters removed. +# The default value is: DOXYGEN. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +MARKDOWN_ID_STYLE = DOXYGEN + # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or @@ -361,8 +392,8 @@ AUTOLINK_SUPPORT = YES # to include (a tag file for) the STL sources as input, then you should set this # tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. +# versus func(std::string) {}). This also makes the inheritance and +# collaboration diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = NO @@ -374,9 +405,9 @@ BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. +# https://www.riverbankcomputing.com/software) sources only. Doxygen will parse +# them like normal C++ but will assume all classes use public instead of private +# inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO @@ -460,7 +491,7 @@ TYPEDEF_HIDES_STRUCT = YES LOOKUP_CACHE_SIZE = 0 -# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use # during processing. When set to 0 doxygen will based this on the number of # cores available in the system. You can set it explicitly to a value larger # than 0 to get more control over the balance between CPU load and processing @@ -473,6 +504,14 @@ LOOKUP_CACHE_SIZE = 0 NUM_PROC_THREADS = 1 +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = NO + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -554,7 +593,8 @@ HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. +# will also hide undocumented C++ concepts if enabled. This option has no effect +# if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO @@ -585,14 +625,15 @@ INTERNAL_DOCS = NO # filesystem is case sensitive (i.e. it supports files in the same directory # whose names only differ in casing), the option must be set to YES to properly # deal with such files in case they appear in the input. For filesystems that -# are not case sensitive the option should be be set to NO to properly deal with +# are not case sensitive the option should be set to NO to properly deal with # output files written for symbols that only differ in casing, such as for two # classes, one named CLASS and the other named Class, and to also support # references to files without having to specify the exact matching casing. On # Windows (including Cygwin) and MacOS, users should typically set this option # to NO, whereas on Linux or other Unix flavors it should typically be set to # YES. -# The default value is: system dependent. +# Possible values are: SYSTEM, NO and YES. +# The default value is: SYSTEM. CASE_SENSE_NAMES = NO @@ -610,6 +651,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -767,7 +814,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -813,27 +861,50 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = NO +# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, doxygen will warn about +# undocumented enumeration values. If set to NO, doxygen will accept +# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: NO. + +WARN_IF_UNDOC_ENUM_VAL = NO + # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when # a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS # then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but # at the end of the doxygen process doxygen will return with a non-zero status. -# Possible values are: NO, YES and FAIL_ON_WARNINGS. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO @@ -844,13 +915,27 @@ WARN_AS_ERROR = NO # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -871,10 +956,21 @@ INPUT = mimalloc-doc.h # libiconv (or the iconv built into libc) for the transcoding. See the libiconv # documentation (see: # https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 +# This tag can be used to specify the character encoding of the source files +# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify +# character encoding on a per file pattern basis. Doxygen will compare the file +# name with each pattern and apply the encoding instead of the default +# INPUT_ENCODING) if there is a match. The character encodings are a list of the +# form: pattern=encoding (like *.php=ISO-8859-1). +# See also: INPUT_ENCODING for further information on supported encodings. + +INPUT_FILE_ENCODING = + # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. @@ -886,12 +982,12 @@ INPUT_ENCODING = UTF-8 # Note the list of default checked file patterns might differ from the list of # default file extension mappings. # -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, -# *.ucf, *.qsf and *.ice. +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cxxm, +# *.cpp, *.cppm, *.ccm, *.c++, *.c++m, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, +# *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, *.h++, *.ixx, *.l, *.cs, *.d, +# *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to +# be provided as doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.cc \ @@ -973,10 +1069,7 @@ EXCLUDE_PATTERNS = # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* +# ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = @@ -1021,6 +1114,11 @@ IMAGE_PATH = # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # +# Note that doxygen will use the data processed and written to standard output +# for further processing, therefore nothing else, like debug statements or used +# commands (so in case of a Windows batch file always use @echo OFF), should be +# written to standard output. +# # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. @@ -1062,6 +1160,15 @@ FILTER_SOURCE_PATTERNS = USE_MDFILE_AS_MAINPAGE = +# The Fortran standard specifies that for fixed formatted Fortran code all +# characters from position 72 are to be considered as comment. A common +# extension is to allow longer lines before the automatic comment starts. The +# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can +# be processed before the automatic comment starts. +# Minimum value: 7, maximum value: 10000, default value: 72. + +FORTRAN_COMMENT_AFTER = 72 + #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- @@ -1076,7 +1183,8 @@ USE_MDFILE_AS_MAINPAGE = SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. +# multi-line macros, enums or list initialized variables directly into the +# documentation. # The default value is: NO. INLINE_SOURCES = NO @@ -1159,9 +1267,11 @@ VERBATIM_HEADERS = YES CLANG_ASSISTED_PARSING = NO -# If clang assisted parsing is enabled and the CLANG_ADD_INC_PATHS tag is set to -# YES then doxygen will add the directory of each input to the include path. +# If the CLANG_ASSISTED_PARSING tag is set to YES and the CLANG_ADD_INC_PATHS +# tag is set to YES then doxygen will add the directory of each input to the +# include path. # The default value is: YES. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. CLANG_ADD_INC_PATHS = YES @@ -1197,10 +1307,11 @@ CLANG_DATABASE_PATH = ALPHABETICAL_INDEX = YES -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. +# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) +# that should be ignored while generating the index headers. The IGNORE_PREFIX +# tag works for classes, function and member names. The entity will be placed in +# the alphabetical list under the first letter of the entity name that remains +# after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = @@ -1279,7 +1390,12 @@ HTML_STYLESHEET = # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. +# list). +# Note: Since the styling of scrollbars can currently not be overruled in +# Webkit/Chromium, the styling will be left out of the default doxygen.css if +# one or more extra stylesheets have been specified. So if scrollbar +# customization is desired it has to be added explicitly. For an example see the +# documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = mimalloc-doxygen.css @@ -1294,9 +1410,22 @@ HTML_EXTRA_STYLESHEET = mimalloc-doxygen.css HTML_EXTRA_FILES = +# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output +# should be rendered with a dark or light theme. +# Possible values are: LIGHT always generates light mode output, DARK always +# generates dark mode output, AUTO_LIGHT automatically sets the mode according +# to the user preference, uses light mode if no preference is set (the default), +# AUTO_DARK automatically sets the mode according to the user preference, uses +# dark mode if no preference is set and TOGGLE allows a user to switch between +# light and dark mode via a button. +# The default value is: AUTO_LIGHT. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE = LIGHT + # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1306,7 +1435,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 189 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1324,15 +1453,6 @@ HTML_COLORSTYLE_SAT = 12 HTML_COLORSTYLE_GAMMA = 240 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via JavaScript. If disabled, the navigation index will @@ -1352,6 +1472,33 @@ HTML_DYNAMIC_MENUS = NO HTML_DYNAMIC_SECTIONS = NO +# If the HTML_CODE_FOLDING tag is set to YES then classes and functions can be +# dynamically folded and expanded in the generated HTML source code. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_CODE_FOLDING = YES + +# If the HTML_COPY_CLIPBOARD tag is set to YES then doxygen will show an icon in +# the top right corner of code and text fragments that allows the user to copy +# its content to the clipboard. Note this only works if supported by the browser +# and the web page is served via a secure context (see: +# https://www.w3.org/TR/secure-contexts/), i.e. using the https: or file: +# protocol. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COPY_CLIPBOARD = YES + +# Doxygen stores a couple of settings persistently in the browser (via e.g. +# cookies). By default these settings apply to all HTML pages generated by +# doxygen across all projects. The HTML_PROJECT_COOKIE tag can be used to store +# the settings under a project specific key, such that the user preferences will +# be stored separately. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_PROJECT_COOKIE = + # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to @@ -1388,6 +1535,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1413,8 +1567,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: -# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1471,6 +1629,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -1573,16 +1741,28 @@ DISABLE_INDEX = YES # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1607,6 +1787,13 @@ TREEVIEW_WIDTH = 180 EXT_LINKS_IN_WINDOW = NO +# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + # If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg # tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see # https://inkscape.org) to generate formulas as SVG images instead of PNGs for @@ -1627,17 +1814,6 @@ HTML_FORMULA_FORMAT = png FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - # The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands # to create new LaTeX commands to be used in formulas as building blocks. See # the section "Including formulas" for details. @@ -1655,11 +1831,29 @@ FORMULA_MACROFILE = USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for MathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1672,15 +1866,21 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see +# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = @@ -1860,29 +2060,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1925,10 +2127,16 @@ PDF_HYPERLINKS = YES USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# The LATEX_BATCHMODE tag signals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1941,16 +2149,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1959,14 +2157,6 @@ LATEX_SOURCE_CODE = NO LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_TIMESTAMP = NO - # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) # path from which the emoji images will be read. If a relative path is entered, # it will be relative to the LATEX_OUTPUT directory. If left blank the @@ -2031,15 +2221,13 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. +# The RTF_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the RTF_OUTPUT output directory. +# Note that the files will be copied as-is; there are no commands or markers +# available. # This tag requires that the tag GENERATE_RTF is set to YES. -RTF_SOURCE_CODE = NO +RTF_EXTRA_FILES = #--------------------------------------------------------------------------- # Configuration options related to the man page output @@ -2137,21 +2325,12 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures # the structure of the code including all documentation. Note that this feature # is still experimental and incomplete at the moment. # The default value is: NO. @@ -2162,6 +2341,28 @@ GENERATE_AUTOGEN_DEF = NO # Configuration options related to Sqlite3 output #--------------------------------------------------------------------------- +# If the GENERATE_SQLITE3 tag is set to YES doxygen will generate a Sqlite3 +# database with symbols found by doxygen stored in tables. +# The default value is: NO. + +GENERATE_SQLITE3 = NO + +# The SQLITE3_OUTPUT tag is used to specify where the Sqlite3 database will be +# put. If a relative path is entered the value of OUTPUT_DIRECTORY will be put +# in front of it. +# The default directory is: sqlite3. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_OUTPUT = sqlite3 + +# The SQLITE3_RECREATE_DB tag is set to YES, the existing doxygen_sqlite3.db +# database file will be recreated with each doxygen run. If set to NO, doxygen +# will warn if a database file is already found and not modify it. +# The default value is: YES. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_RECREATE_DB = YES + #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- @@ -2236,7 +2437,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -2303,15 +2505,15 @@ TAGFILES = GENERATE_TAGFILE = -# If the ALLEXTERNALS tag is set to YES, all external class will be listed in -# the class index. If set to NO, only the inherited external classes will be -# listed. +# If the ALLEXTERNALS tag is set to YES, all external classes and namespaces +# will be listed in the class and namespace index. If set to NO, only the +# inherited external classes will be listed. # The default value is: NO. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will be +# in the topic index. If set to NO, only the current project's groups will be # listed. # The default value is: YES. @@ -2325,25 +2527,9 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. @@ -2352,7 +2538,7 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: NO. @@ -2369,49 +2555,77 @@ HAVE_DOT = NO DOT_NUM_THREADS = 0 -# When you want a differently looking font in the dot files that doxygen -# generates you can specify the font name using DOT_FONTNAME. You need to make -# sure dot is able to find the font, which can be done by putting it in a -# standard location or by setting the DOTFONTPATH environment variable or by -# setting DOT_FONTPATH to the directory containing the font. -# The default value is: Helvetica. +# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of +# subgraphs. When you want a differently looking font in the dot files that +# doxygen generates you can specify fontname, fontcolor and fontsize attributes. +# For details please see Node, +# Edge and Graph Attributes specification You need to make sure dot is able +# to find the font, which can be done by putting it in a standard location or by +# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. Default graphviz fontsize is 14. +# The default value is: fontname=Helvetica,fontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Helvetica +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" -# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of -# dot graphs. -# Minimum value: 4, maximum value: 24, default value: 10. +# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can +# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about +# arrows shapes. +# The default value is: labelfontname=Helvetica,labelfontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 10 +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" -# By default doxygen will tell dot to use the default font as specified with -# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set -# the path where dot can find it using this tag. +# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes +# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification +# The default value is: shape=box,height=0.2,width=0.4. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" + +# You can set the path where dot can find font specified with fontname in +# DOT_COMMON_ATTR and others dot attributes. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. Explicit enabling an inheritance +# graph or choosing a different representation for an inheritance graph of a +# specific class, can be accomplished by means of the command \inheritancegraph. +# Disabling an inheritance graph can be accomplished by means of the command +# \hideinheritancegraph. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a # graph for each documented class showing the direct and indirect implementation # dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. +# class with other documented classes. Explicit enabling a collaboration graph, +# when COLLABORATION_GRAPH is set to NO, can be accomplished by means of the +# command \collaborationgraph. Disabling a collaboration graph can be +# accomplished by means of the command \hidecollaborationgraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# groups, showing the direct groups dependencies. Explicit enabling a group +# dependency graph, when GROUP_GRAPHS is set to NO, can be accomplished by means +# of the command \groupgraph. Disabling a directory graph can be accomplished by +# means of the command \hidegroupgraph. See also the chapter Grouping in the +# manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2453,8 +2667,8 @@ DOT_UML_DETAILS = NO # The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters # to display on a single line. If the actual line length exceeds this threshold -# significantly it will wrapped across multiple lines. Some heuristics are apply -# to avoid ugly line breaks. +# significantly it will be wrapped across multiple lines. Some heuristics are +# applied to avoid ugly line breaks. # Minimum value: 0, maximum value: 1000, default value: 17. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2471,7 +2685,9 @@ TEMPLATE_RELATIONS = NO # If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to # YES then doxygen will generate a graph for each documented file showing the # direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an include graph, when INCLUDE_GRAPH is is set to NO, +# can be accomplished by means of the command \includegraph. Disabling an +# include graph can be accomplished by means of the command \hideincludegraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2480,7 +2696,10 @@ INCLUDE_GRAPH = YES # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are # set to YES then doxygen will generate a graph for each documented file showing # the direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an included by graph, when INCLUDED_BY_GRAPH is set +# to NO, can be accomplished by means of the command \includedbygraph. Disabling +# an included by graph can be accomplished by means of the command +# \hideincludedbygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2520,16 +2739,26 @@ GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the # dependencies a directory has on other directories in a graphical way. The # dependency relations are determined by the #include relations between the -# files in the directories. +# files in the directories. Explicit enabling a directory graph, when +# DIRECTORY_GRAPH is set to NO, can be accomplished by means of the command +# \directorygraph. Disabling a directory graph can be accomplished by means of +# the command \hidedirectorygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). +# https://www.graphviz.org/)). # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). @@ -2566,11 +2795,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2579,10 +2809,10 @@ MSCFILE_DIRS = DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = @@ -2599,7 +2829,7 @@ PLANTUML_INCLUDE_PATH = # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes # that will be shown in the graph. If the number of nodes in a graph becomes # larger than this value, doxygen will truncate the graph, which is visualized -# by representing a node as a red box. Note that doxygen if the number of direct +# by representing a node as a red box. Note that if the number of direct # children of the root node in a graph is already larger than # DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that # the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. @@ -2620,18 +2850,6 @@ DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not seem -# to support this out of the box. -# -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_TRANSPARENT = NO - # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support @@ -2644,6 +2862,8 @@ DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2652,8 +2872,24 @@ GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. # -# Note: This setting is not only used for dot files but also for msc and -# plantuml temporary files. +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. DOT_CLEANUP = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/third-party/mimalloc/doc/mimalloc-doc.h b/third-party/mimalloc/doc/mimalloc-doc.h index d79eb2f866..753c062f4a 100644 --- a/third-party/mimalloc/doc/mimalloc-doc.h +++ b/third-party/mimalloc/doc/mimalloc-doc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +Copyright (c) 2018-2025, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -25,12 +25,15 @@ without code changes, for example, on Unix you can use it as: ``` Notable aspects of the design include: - - __small and consistent__: the library is about 8k LOC using simple and consistent data structures. This makes it very suitable to integrate and adapt in other projects. For runtime systems it provides hooks for a monotonic _heartbeat_ and deferred freeing (for bounded worst-case times with reference counting). + Partly due to its simplicity, mimalloc has been ported to many systems (Windows, macOS, + Linux, WASM, various BSD's, Haiku, MUSL, etc) and has excellent support for dynamic overriding. + At the same time, it is an industrial strength allocator that runs (very) large scale + distributed services on thousands of machines with excellent worst case latencies. - __free list sharding__: instead of one big free list (per size class) we have many smaller lists per "mimalloc page" which reduces fragmentation and increases locality -- @@ -45,23 +48,23 @@ Notable aspects of the design include: and the chance of contending on a single location will be low -- this is quite similar to randomized algorithms like skip lists where adding a random oracle removes the need for a more complex algorithm. -- __eager page reset__: when a "page" becomes empty (with increased chance - due to free list sharding) the memory is marked to the OS as unused ("reset" or "purged") +- __eager page purging__: when a "page" becomes empty (with increased chance + due to free list sharding) the memory is marked to the OS as unused (reset or decommitted) reducing (real) memory pressure and fragmentation, especially in long running programs. -- __secure__: _mimalloc_ can be build in secure mode, adding guard pages, +- __secure__: _mimalloc_ can be built in secure mode, adding guard pages, randomized allocation, encrypted free lists, etc. to protect against various - heap vulnerabilities. The performance penalty is only around 5% on average + heap vulnerabilities. The performance penalty is usually around 10% on average over our benchmarks. - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions. A heap can be destroyed at once instead of deallocating each object separately. - __bounded__: it does not suffer from _blowup_ \[1\], has bounded worst-case allocation - times (_wcat_), bounded space overhead (~0.2% meta-data, with low internal fragmentation), - and has no internal points of contention using only atomic operations. -- __fast__: In our benchmarks (see [below](#performance)), - _mimalloc_ outperforms all other leading allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc), - and usually uses less memory (up to 25% more in the worst case). A nice property - is that it does consistently well over a wide range of benchmarks. + times (_wcat_) (upto OS primitives), bounded space overhead (~0.2% meta-data, with low + internal fragmentation), and has no internal points of contention using only atomic operations. +- __fast__: In our benchmarks (see [below](#bench)), + _mimalloc_ outperforms other leading allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc), + and often uses less memory. A nice property is that it does consistently well over a wide range + of benchmarks. There is also good huge OS page support for larger server programs. You can read more on the design of _mimalloc_ in the [technical report](https://www.microsoft.com/en-us/research/publication/mimalloc-free-list-sharding-in-action) @@ -278,8 +281,7 @@ void* mi_zalloc_small(size_t size); /// The returned size can be /// used to call \a mi_expand successfully. /// The returned size is always at least equal to the -/// allocated size of \a p, and, in the current design, -/// should be less than 16.7% more. +/// allocated size of \a p. /// /// @see [_msize](https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/msize?view=vs-2017) (Windows) /// @see [malloc_usable_size](http://man7.org/linux/man-pages/man3/malloc_usable_size.3.html) (Linux) @@ -304,7 +306,7 @@ size_t mi_good_size(size_t size); /// in very narrow circumstances; in particular, when a long running thread /// allocates a lot of blocks that are freed by other threads it may improve /// resource usage by calling this every once in a while. -void mi_collect(bool force); +void mi_collect(bool force); /// Deprecated /// @param out Ignored, outputs to the registered output function or stderr by default. @@ -428,7 +430,7 @@ int mi_reserve_os_memory(size_t size, bool commit, bool allow_large); /// allocated in some manner and available for use my mimalloc. /// @param start Start of the memory area /// @param size The size of the memory area. -/// @param commit Is the area already committed? +/// @param is_committed Is the area already committed? /// @param is_large Does it consist of large OS pages? Set this to \a true as well for memory /// that should not be decommitted or protected (like rdma etc.) /// @param is_zero Does the area consists of zero's? @@ -453,7 +455,7 @@ int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t /// Reserve \a pages of huge OS pages (1GiB) at a specific \a numa_node, /// but stops after at most `timeout_msecs` seconds. /// @param pages The number of 1GiB pages to reserve. -/// @param numa_node The NUMA node where the memory is reserved (start at 0). +/// @param numa_node The NUMA node where the memory is reserved (start at 0). Use -1 for no affinity. /// @param timeout_msecs Maximum number of milli-seconds to try reserving, or 0 for no timeout. /// @returns 0 if successful, \a ENOMEM if running out of memory, or \a ETIMEDOUT if timed out. /// @@ -486,6 +488,91 @@ bool mi_is_redirected(); /// on other systems as the amount of read/write accessible memory reserved by mimalloc. void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults); +/// @brief Show all current arena's. +/// @param show_inuse Show the arena blocks that are in use. +/// @param show_abandoned Show the abandoned arena blocks. +/// @param show_purge Show arena blocks scheduled for purging. +void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge); + +/// Mimalloc uses large (virtual) memory areas, called "arena"s, from the OS to manage its memory. +/// Each arena has an associated identifier. +typedef int mi_arena_id_t; + +/// @brief Return the size of an arena. +/// @param arena_id The arena identifier. +/// @param size Returned size in bytes of the (virtual) arena area. +/// @return base address of the arena. +void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); + +/// @brief Reserve huge OS pages (1GiB) into a single arena. +/// @param pages Number of 1GiB pages to reserve. +/// @param numa_node The associated NUMA node, or -1 for no NUMA preference. +/// @param timeout_msecs Max amount of milli-seconds this operation is allowed to take. (0 is infinite) +/// @param exclusive If exclusive, only a heap associated with this arena can allocate in it. +/// @param arena_id The arena identifier. +/// @return 0 if successful, \a ENOMEM if running out of memory, or \a ETIMEDOUT if timed out. +int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id); + +/// @brief Reserve OS memory to be managed in an arena. +/// @param size Size the reserve. +/// @param commit Should the memory be initially committed? +/// @param allow_large Allow the use of large OS pages? +/// @param exclusive Is the returned arena exclusive? +/// @param arena_id The new arena identifier. +/// @return Zero on success, an error code otherwise. +int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id); + +/// @brief Manage externally allocated memory as a mimalloc arena. This memory will not be freed by mimalloc. +/// @param start Start address of the area. +/// @param size Size in bytes of the area. +/// @param is_committed Is the memory already committed? +/// @param is_large Does it consist of (pinned) large OS pages? +/// @param is_zero Is the memory zero-initialized? +/// @param numa_node Associated NUMA node, or -1 to have no NUMA preference. +/// @param exclusive Is the arena exclusive (where only heaps associated with the arena can allocate in it) +/// @param arena_id The new arena identifier. +/// @return `true` if successful. +bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id); + +/// @brief Create a new heap that only allocates in the specified arena. +/// @param arena_id The arena identifier. +/// @return The new heap or `NULL`. +mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); + +/// @brief Create a new heap +/// @param heap_tag The heap tag associated with this heap; heaps only reclaim memory between heaps with the same tag. +/// @param allow_destroy Is \a mi_heap_destroy allowed? Not allowing this allows the heap to reclaim memory from terminated threads. +/// @param arena_id If not 0, the heap will only allocate from the specified arena. +/// @return A new heap or `NULL` on failure. +/// +/// The \a arena_id can be used by runtimes to allocate only in a specified pre-reserved arena. +/// This is used for example for a compressed pointer heap in Koka. +/// The \a heap_tag enables heaps to keep objects of a certain type isolated to heaps with that tag. +/// This is used for example in the CPython integration. +mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id); + +/// A process can associate threads with sub-processes. +/// A sub-process will not reclaim memory from (abandoned heaps/threads) +/// other subprocesses. +typedef void* mi_subproc_id_t; + +/// @brief Get the main sub-process identifier. +mi_subproc_id_t mi_subproc_main(void); + +/// @brief Create a fresh sub-process (with no associated threads yet). +/// @return The new sub-process identifier. +mi_subproc_id_t mi_subproc_new(void); + +/// @brief Delete a previously created sub-process. +/// @param subproc The sub-process identifier. +/// Only delete sub-processes if all associated threads have terminated. +void mi_subproc_delete(mi_subproc_id_t subproc); + +/// Add the current thread to the given sub-process. +/// This should be called right after a thread is created (and no allocation has taken place yet) +void mi_subproc_add_current_thread(mi_subproc_id_t subproc); + + /// \} // ------------------------------------------------------ @@ -495,20 +582,24 @@ void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_m /// \defgroup aligned Aligned Allocation /// /// Allocating aligned memory blocks. +/// Note that `alignment` always follows `size` for consistency with the unaligned +/// allocation API, but unfortunately this differs from `posix_memalign` and `aligned_alloc` in the C library. /// /// \{ -/// The maximum supported alignment size (currently 1MiB). -#define MI_BLOCK_ALIGNMENT_MAX (1024*1024UL) - /// Allocate \a size bytes aligned by \a alignment. /// @param size number of bytes to allocate. -/// @param alignment the minimal alignment of the allocated memory. Must be less than #MI_BLOCK_ALIGNMENT_MAX. -/// @returns pointer to the allocated memory or \a NULL if out of memory. -/// The returned pointer is aligned by \a alignment, i.e. -/// `(uintptr_t)p % alignment == 0`. -/// +/// @param alignment the minimal alignment of the allocated memory. +/// @returns pointer to the allocated memory or \a NULL if out of memory, +/// or if the alignment is not a power of 2 (including 0). The \a size is unrestricted +/// (and does not have to be an integral multiple of the \a alignment). +/// The returned pointer is aligned by \a alignment, i.e. `(uintptr_t)p % alignment == 0`. /// Returns a unique pointer if called with \a size 0. +/// +/// Note that `alignment` always follows `size` for consistency with the unaligned +/// allocation API, but unfortunately this differs from `posix_memalign` and `aligned_alloc` in the C library. +/// +/// @see [aligned_alloc](https://en.cppreference.com/w/c/memory/aligned_alloc) (in the standard C11 library, with switched arguments!) /// @see [_aligned_malloc](https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/aligned-malloc?view=vs-2017) (on Windows) /// @see [aligned_alloc](http://man.openbsd.org/reallocarray) (on BSD, with switched arguments!) /// @see [posix_memalign](https://linux.die.net/man/3/posix_memalign) (on Posix, with switched arguments!) @@ -522,11 +613,12 @@ void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment); /// @param size number of bytes to allocate. /// @param alignment the minimal alignment of the allocated memory at \a offset. /// @param offset the offset that should be aligned. -/// @returns pointer to the allocated memory or \a NULL if out of memory. -/// The returned pointer is aligned by \a alignment at \a offset, i.e. -/// `((uintptr_t)p + offset) % alignment == 0`. -/// +/// @returns pointer to the allocated memory or \a NULL if out of memory, +/// or if the alignment is not a power of 2 (including 0). The \a size is unrestricted +/// (and does not have to be an integral multiple of the \a alignment). +/// The returned pointer is aligned by \a alignment, i.e. `(uintptr_t)p % alignment == 0`. /// Returns a unique pointer if called with \a size 0. +/// /// @see [_aligned_offset_malloc](https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/aligned-offset-malloc?view=vs-2017) (on Windows) void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset); void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset); @@ -574,12 +666,12 @@ void mi_heap_delete(mi_heap_t* heap); /// heap is set to the backing heap. void mi_heap_destroy(mi_heap_t* heap); -/// Set the default heap to use for mi_malloc() et al. +/// Set the default heap to use in the current thread for mi_malloc() et al. /// @param heap The new default heap. /// @returns The previous default heap. mi_heap_t* mi_heap_set_default(mi_heap_t* heap); -/// Get the default heap that is used for mi_malloc() et al. +/// Get the default heap that is used for mi_malloc() et al. (for the current thread). /// @returns The current default heap. mi_heap_t* mi_heap_get_default(); @@ -764,6 +856,8 @@ typedef struct mi_heap_area_s { size_t committed; ///< current committed bytes of this area size_t used; ///< bytes in use by allocated blocks size_t block_size; ///< size in bytes of one block + size_t full_block_size; ///< size in bytes of a full block including padding and metadata. + int heap_tag; ///< heap tag associated with this area (see \a mi_heap_new_ex) } mi_heap_area_t; /// Visitor function passed to mi_heap_visit_blocks() @@ -788,6 +882,23 @@ typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* a /// @returns \a true if all areas and blocks were visited. bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); +/// @brief Visit all areas and blocks in abandoned heaps. +/// @param subproc_id The sub-process id associated with the abandoned heaps. +/// @param heap_tag Visit only abandoned memory with the specified heap tag, use -1 to visit all abandoned memory. +/// @param visit_blocks If \a true visits all allocated blocks, otherwise +/// \a visitor is only called for every heap area. +/// @param visitor This function is called for every area in the heap +/// (with \a block as \a NULL). If \a visit_all_blocks is +/// \a true, \a visitor is also called for every allocated +/// block in every area (with `block!=NULL`). +/// return \a false from this function to stop visiting early. +/// @param arg extra argument passed to the \a visitor. +/// @return \a true if all areas and blocks were visited. +/// +/// Note: requires the option `mi_option_visit_abandoned` to be set +/// at the start of the program. +bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + /// \} /// \defgroup options Runtime Options @@ -799,34 +910,38 @@ bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block /// Runtime options. typedef enum mi_option_e { // stable options - mi_option_show_errors, ///< Print error messages to `stderr`. - mi_option_show_stats, ///< Print statistics to `stderr` when the program is done. - mi_option_verbose, ///< Print verbose messages to `stderr`. - - // the following options are experimental - mi_option_eager_commit, ///< Eagerly commit segments (4MiB) (enabled by default). - mi_option_large_os_pages, ///< Use large OS pages (2MiB in size) if possible - mi_option_reserve_huge_os_pages, ///< The number of huge OS pages (1GiB in size) to reserve at the start of the program. - mi_option_reserve_huge_os_pages_at, ///< Reserve huge OS pages at node N. - mi_option_reserve_os_memory, ///< Reserve specified amount of OS memory at startup, e.g. "1g" or "512m". - mi_option_segment_cache, ///< The number of segments per thread to keep cached (0). - mi_option_page_reset, ///< Reset page memory after \a mi_option_reset_delay milliseconds when it becomes free. - mi_option_abandoned_page_reset, //< Reset free page memory when a thread terminates. - mi_option_use_numa_nodes, ///< Pretend there are at most N NUMA nodes; Use 0 to use the actual detected NUMA nodes at runtime. - mi_option_eager_commit_delay, ///< the first N segments per thread are not eagerly committed (=1). - mi_option_os_tag, ///< OS tag to assign to mimalloc'd memory - mi_option_limit_os_alloc, ///< If set to 1, do not use OS memory for allocation (but only pre-reserved arenas) - - // v1.x specific options - mi_option_eager_region_commit, ///< Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows) - mi_option_segment_reset, ///< Experimental - mi_option_reset_delay, ///< Delay in milli-seconds before resetting a page (100ms by default) - mi_option_purge_decommits, ///< Experimental - - // v2.x specific options - mi_option_allow_purge, ///< Enable decommitting memory (=on) - mi_option_purge_delay, ///< Decommit page memory after N milli-seconds delay (25ms). - mi_option_segment_purge_delay, ///< Decommit large segment memory after N milli-seconds delay (500ms). + mi_option_show_errors, ///< Print error messages. + mi_option_show_stats, ///< Print statistics on termination. + mi_option_verbose, ///< Print verbose messages. + mi_option_max_errors, ///< issue at most N error messages + mi_option_max_warnings, ///< issue at most N warning messages + + // advanced options + mi_option_reserve_huge_os_pages, ///< reserve N huge OS pages (1GiB pages) at startup + mi_option_reserve_huge_os_pages_at, ///< Reserve N huge OS pages at a specific NUMA node N. + mi_option_reserve_os_memory, ///< reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`) + mi_option_allow_large_os_pages, ///< allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process. + mi_option_purge_decommits, ///< should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit) + mi_option_arena_reserve, ///< initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`) + mi_option_os_tag, ///< tag used for OS logging (macOS only for now) (=100) + mi_option_retry_on_oom, ///< retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) + + // experimental options + mi_option_eager_commit, ///< eager commit segments? (after `eager_commit_delay` segments) (enabled by default). + mi_option_eager_commit_delay, ///< the first N segments per thread are not eagerly committed (but per page in the segment on demand) + mi_option_arena_eager_commit, ///< eager commit arenas? Use 2 to enable just on overcommit systems (=2) + mi_option_abandoned_page_purge, ///< immediately purge delayed purges on thread termination + mi_option_purge_delay, ///< memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10) + mi_option_use_numa_nodes, ///< 0 = use all available numa nodes, otherwise use at most N nodes. + mi_option_disallow_os_alloc, ///< 1 = do not use OS memory for allocation (but only programmatically reserved arenas) + mi_option_limit_os_alloc, ///< If set to 1, do not use OS memory for allocation (but only pre-reserved arenas) + mi_option_max_segment_reclaim, ///< max. percentage of the abandoned segments can be reclaimed per try (=10%) + mi_option_destroy_on_exit, ///< if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe + mi_option_arena_purge_mult, ///< multiplier for `purge_delay` for the purging delay for arenas (=10) + mi_option_abandoned_reclaim_on_free, ///< allow to reclaim an abandoned segment on a free (=1) + mi_option_purge_extend_delay, ///< extend purge delay on each subsequent delay (=1) + mi_option_disallow_arena_alloc, ///< 1 = do not use arena's for allocation (except if using specific arena id's) + mi_option_visit_abandoned, ///< allow visiting heap blocks from abandoned threads (=0) _mi_option_last } mi_option_t; @@ -838,7 +953,10 @@ void mi_option_disable(mi_option_t option); void mi_option_set_enabled(mi_option_t option, bool enable); void mi_option_set_enabled_default(mi_option_t option, bool enable); -long mi_option_get(mi_option_t option); +long mi_option_get(mi_option_t option); +long mi_option_get_clamp(mi_option_t option, long min, long max); +size_t mi_option_get_size(mi_option_t option); + void mi_option_set(mi_option_t option, long value); void mi_option_set_default(mi_option_t option, long value); @@ -852,21 +970,27 @@ void mi_option_set_default(mi_option_t option, long value); /// /// \{ +/// Just as `free` but also checks if the pointer `p` belongs to our heap. +void mi_cfree(void* p); +void* mi__expand(void* p, size_t newsize); + void* mi_recalloc(void* p, size_t count, size_t size); size_t mi_malloc_size(const void* p); +size_t mi_malloc_good_size(size_t size); size_t mi_malloc_usable_size(const void *p); -/// Just as `free` but also checks if the pointer `p` belongs to our heap. -void mi_cfree(void* p); - int mi_posix_memalign(void** p, size_t alignment, size_t size); int mi__posix_memalign(void** p, size_t alignment, size_t size); void* mi_memalign(size_t alignment, size_t size); void* mi_valloc(size_t size); - void* mi_pvalloc(size_t size); void* mi_aligned_alloc(size_t alignment, size_t size); +unsigned short* mi_wcsdup(const unsigned short* s); +unsigned char* mi_mbsdup(const unsigned char* s); +int mi_dupenv_s(char** buf, size_t* size, const char* name); +int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name); + /// Correspond s to [reallocarray](https://www.freebsd.org/cgi/man.cgi?query=reallocarray&sektion=3&manpath=freebsd-release-ports) /// in FreeBSD. void* mi_reallocarray(void* p, size_t count, size_t size); @@ -874,6 +998,9 @@ void* mi_reallocarray(void* p, size_t count, size_t size); /// Corresponds to [reallocarr](https://man.netbsd.org/reallocarr.3) in NetBSD. int mi_reallocarr(void* p, size_t count, size_t size); +void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment); +void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset); + void mi_free_size(void* p, size_t size); void mi_free_size_aligned(void* p, size_t size, size_t alignment); void mi_free_aligned(void* p, size_t alignment); @@ -998,7 +1125,7 @@ mimalloc uses only safe OS calls (`mmap` and `VirtualAlloc`) and can co-exist with other allocators linked to the same program. If you use `cmake`, you can simply use: ``` -find_package(mimalloc 1.0 REQUIRED) +find_package(mimalloc 2.1 REQUIRED) ``` in your `CMakeLists.txt` to find a locally installed mimalloc. Then use either: ``` @@ -1012,7 +1139,7 @@ to link with the static library. See `test\CMakeLists.txt` for an example. ### C++ For best performance in C++ programs, it is also recommended to override the -global `new` and `delete` operators. For convience, mimalloc provides +global `new` and `delete` operators. For convenience, mimalloc provides [`mimalloc-new-delete.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) which does this for you -- just include it in a single(!) source file in your project. In C++, mimalloc also provides the `mi_stl_allocator` struct which implements the `std::allocator` @@ -1071,38 +1198,64 @@ See \ref overrides for more info. /*! \page environment Environment Options -You can set further options either programmatically (using [`mi_option_set`](https://microsoft.github.io/mimalloc/group__options.html)), -or via environment variables. +You can set further options either programmatically (using [`mi_option_set`](https://microsoft.github.io/mimalloc/group__options.html)), or via environment variables: - `MIMALLOC_SHOW_STATS=1`: show statistics when the program terminates. - `MIMALLOC_VERBOSE=1`: show verbose messages. - `MIMALLOC_SHOW_ERRORS=1`: show error and warning messages. -- `MIMALLOC_PAGE_RESET=0`: by default, mimalloc will reset (or purge) OS pages when not in use to signal to the OS - that the underlying physical memory can be reused. This can reduce memory fragmentation in long running (server) - programs. By setting it to `0` no such page resets will be done which can improve performance for programs that are not long - running. As an alternative, the `MIMALLOC_DECOMMIT_DELAY=` can be set higher (100ms by default) to make the page - reset occur less frequently instead of turning it off completely. -- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages (2MiB) when available; for some workloads this can significantly - improve performance. Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs - to explicitly allow large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes + +Advanced options: + +- `MIMALLOC_ARENA_EAGER_COMMIT=2`: turns on eager commit for the large arenas (usually 1GiB) from which mimalloc + allocates segments and pages. Set this to 2 (default) to + only enable this on overcommit systems (e.g. Linux). Set this to 1 to enable explicitly on other systems + as well (like Windows or macOS) which may improve performance (as the whole arena is committed at once). + Note that eager commit only increases the commit but not the actual the peak resident set + (rss) so it is generally ok to enable this. +- `MIMALLOC_PURGE_DELAY=N`: the delay in `N` milli-seconds (by default `10`) after which mimalloc will purge + OS pages that are not in use. This signals to the OS that the underlying physical memory can be reused which + can reduce memory fragmentation especially in long running (server) programs. Setting `N` to `0` purges immediately when + a page becomes unused which can improve memory usage but also decreases performance. Setting `N` to a higher + value like `100` can improve performance (sometimes by a lot) at the cost of potentially using more memory at times. + Setting it to `-1` disables purging completely. +- `MIMALLOC_PURGE_DECOMMITS=1`: By default "purging" memory means unused memory is decommitted (`MEM_DECOMMIT` on Windows, + `MADV_DONTNEED` (which decresease rss immediately) on `mmap` systems). Set this to 0 to instead "reset" unused + memory on a purge (`MEM_RESET` on Windows, generally `MADV_FREE` (which does not decrease rss immediately) on `mmap` systems). + Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual + address ranges and decommits within those ranges (to make the underlying physical memory available to other processes). + +Further options for large workloads and services: + +- `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected + at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than + the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA + nodes (but this can happen in any case as NUMA local allocation is always a best effort but not guaranteed). +- `MIMALLOC_ALLOW_LARGE_OS_PAGES=1`: use large OS pages (2 or 4MiB) when available; for some workloads this can significantly + improve performance. When this option is disabled (default), it also disables transparent huge pages (THP) for the process + (on Linux and Android). On Linux the default setting is 2 -- this enables the use of large pages through THP only. + Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs + to explicitly give permissions for large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that - can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead when possible). -- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where N is the number of 1GiB _huge_ OS pages. This reserves the huge pages at + can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead whenever possible). +- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where `N` is the number of 1GiB _huge_ OS pages. This reserves the huge pages at startup and sometimes this can give a large (latency) performance improvement on big workloads. - Usually it is better to not use - `MIMALLOC_LARGE_OS_PAGES` in combination with this setting. Just like large OS pages, use with care as reserving + Usually it is better to not use `MIMALLOC_ALLOW_LARGE_OS_PAGES=1` in combination with this setting. Just like large + OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at startup only once). - Note that we usually need to explicitly enable huge OS pages (as on [Windows][windows-huge] and [Linux][linux-huge])). With huge OS pages, it may be beneficial to set the setting + Note that we usually need to explicitly give permission for huge OS pages (as on [Windows][windows-huge] and [Linux][linux-huge])). + With huge OS pages, it may be beneficial to set the setting `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived - and allocate just a little to take up space in the huge OS page area (which cannot be reset). -- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. - (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) + and allocate just a little to take up space in the huge OS page area (which cannot be purged as huge OS pages are pinned + to physical memory). + The huge pages are usually allocated evenly among NUMA nodes. + We can use `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N` where `N` is the numa node (starting at 0) to allocate all + the huge pages at a specific numa node instead. Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the -OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in big increments. +OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments. [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 @@ -1111,88 +1264,106 @@ OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the /*! \page overrides Overriding Malloc -Overriding the standard `malloc` can be done either _dynamically_ or _statically_. +Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. ## Dynamic override This is the recommended way to override the standard malloc interface. +### Dynamic Override on Linux, BSD -### Linux, BSD - -On these systems we preload the mimalloc shared +On these ELF-based systems we preload the mimalloc shared library so all calls to the standard `malloc` interface are resolved to the _mimalloc_ library. - -- `env LD_PRELOAD=/usr/lib/libmimalloc.so myprogram` +``` +> env LD_PRELOAD=/usr/lib/libmimalloc.so myprogram +``` You can set extra environment variables to check that mimalloc is running, like: ``` -env MIMALLOC_VERBOSE=1 LD_PRELOAD=/usr/lib/libmimalloc.so myprogram +> env MIMALLOC_VERBOSE=1 LD_PRELOAD=/usr/lib/libmimalloc.so myprogram ``` or run with the debug version to get detailed statistics: ``` -env MIMALLOC_SHOW_STATS=1 LD_PRELOAD=/usr/lib/libmimalloc-debug.so myprogram +> env MIMALLOC_SHOW_STATS=1 LD_PRELOAD=/usr/lib/libmimalloc-debug.so myprogram ``` -### MacOS +### Dynamic Override on MacOS On macOS we can also preload the mimalloc shared library so all calls to the standard `malloc` interface are resolved to the _mimalloc_ library. - -- `env DYLD_FORCE_FLAT_NAMESPACE=1 DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram` +``` +> env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram +``` Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -(Note: macOS support for dynamic overriding is recent, please report any issues.) +### Dynamic Override on Windows + +Dynamically overriding on mimalloc on Windows +is robust and has the particular advantage to be able to redirect all malloc/free calls +that go through the (dynamic) C runtime allocator, including those from other DLL's or +libraries. As it intercepts all allocation calls on a low level, it can be used reliably +on large programs that include other 3rd party components. +There are four requirements to make the overriding work well: + +1. Use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -### Windows +2. Link your program explicitly with the `mimalloc.lib` export library for the `mimalloc.dll`. + (which must be compiled with `-DMI_OVERRIDE=ON`, which is the default though). + To ensure the `mimalloc.dll` is actually loaded at run-time it is easiest + to insert some call to the mimalloc API in the `main` function, like `mi_version()` + (or use the `/include:mi_version` switch on the linker command, or + similarly, `#pragma comment(linker, "/include:mi_version")` in some source file). + See the `mimalloc-test-override` project for an example on how to use this. -Overriding on Windows is robust and has the -particular advantage to be able to redirect all malloc/free calls that go through -the (dynamic) C runtime allocator, including those from other DLL's or libraries. +3. The `mimalloc-redirect.dll` must be put in the same directory as the main + `mimalloc.dll` at runtime (as it is a dependency of that DLL). + The redirection DLL ensures that all calls to the C runtime malloc API get + redirected to mimalloc functions (which reside in `mimalloc.dll`). -The overriding on Windows requires that you link your program explicitly with -the mimalloc DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be available -in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency). -The redirection DLL ensures that all calls to the C runtime malloc API get redirected to -mimalloc (in `mimalloc-override.dll`). +4. Ensure the `mimalloc.dll` comes as early as possible in the import + list of the final executable (so it can intercept all potential allocations). + You can use `minject -l ` to check this if needed. -To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some -call to the mimalloc API in the `main` function, like `mi_version()` -(or use the `/INCLUDE:mi_version` switch on the linker). See the `mimalloc-override-test` project -for an example on how to use this. For best performance on Windows with C++, it +For best performance on Windows with C++, it is also recommended to also override the `new`/`delete` operations (by including -[`mimalloc-new-delete.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) a single(!) source file in your project). +[`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) +a single(!) source file in your project). The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic -overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. +overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully +redirected. -(Note: in principle, it is possible to even patch existing executables without any recompilation -if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the `mimalloc-override.dll` -into the import table (and put `mimalloc-redirect.dll` in the same folder) -Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388)). +For different platforms than x64, you may need a specific [redirection dll](bin). +Furthermore, we cannot always re-link an executable or ensure `mimalloc.dll` comes +first in the import table. In such cases the [`minject`](bin) tool can be used +to patch the executable's import tables. ## Static override -On Unix systems, you can also statically link with _mimalloc_ to override the standard +On Unix-like systems, you can also statically link with _mimalloc_ to override the standard malloc interface. The recommended way is to link the final program with the -_mimalloc_ single object file (`mimalloc-override.o`). We use +_mimalloc_ single object file (`mimalloc.o`). We use an object file instead of a library file as linkers give preference to that over archives to resolve symbols. To ensure that the standard malloc interface resolves to the _mimalloc_ library, link it as the first object file. For example: - ``` -gcc -o myprogram mimalloc-override.o myfile1.c ... +> gcc -o myprogram mimalloc.o myfile1.c ... ``` +Another way to override statically that works on all platforms, is to +link statically to mimalloc (as shown in the introduction) and include a +header file in each source file that re-defines `malloc` etc. to `mi_malloc`. +This is provided by [`mimalloc-override.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-override.h). This only works reliably though if all sources are +under your control or otherwise mixing of pointers from different heaps may occur! + ## List of Overrides: The specific functions that get redirected to the _mimalloc_ library are: diff --git a/third-party/mimalloc/doc/mimalloc-doxygen.css b/third-party/mimalloc/doc/mimalloc-doxygen.css index b24f564326..c889a8d2c3 100644 --- a/third-party/mimalloc/doc/mimalloc-doxygen.css +++ b/third-party/mimalloc/doc/mimalloc-doxygen.css @@ -47,3 +47,14 @@ div.fragment { #nav-sync img { display: none; } +h1,h2,h3,h4,h5,h6 { + transition:none; +} +.memtitle { + background-image: none; + background-color: #EEE; +} +table.memproto, .memproto { + text-shadow: none; + font-size: 110%; +} diff --git a/third-party/mimalloc/docker/alpine-arm32v7/Dockerfile b/third-party/mimalloc/docker/alpine-arm32v7/Dockerfile index 56f071db3c..f74934fb3f 100644 --- a/third-party/mimalloc/docker/alpine-arm32v7/Dockerfile +++ b/third-party/mimalloc/docker/alpine-arm32v7/Dockerfile @@ -1,10 +1,10 @@ # install from an image -# download first an appropiate tar.gz image into the current directory +# download first an appropriate tar.gz image into the current directory # from: FROM scratch # Substitute the image name that was downloaded -ADD alpine-minirootfs-20240329-armv7.tar.gz / +ADD alpine-minirootfs-20240329-armv7.tar.gz / # Install tools RUN apk add build-base make cmake @@ -15,7 +15,7 @@ RUN mkdir -p /home/dev WORKDIR /home/dev # Get mimalloc -RUN git clone https://github.com/microsoft/mimalloc -b dev-slice +RUN git clone https://github.com/microsoft/mimalloc -b dev2 RUN mkdir -p mimalloc/out/release RUN mkdir -p mimalloc/out/debug diff --git a/third-party/mimalloc/docker/alpine/Dockerfile b/third-party/mimalloc/docker/alpine/Dockerfile index b222b79194..e1234a9b64 100644 --- a/third-party/mimalloc/docker/alpine/Dockerfile +++ b/third-party/mimalloc/docker/alpine/Dockerfile @@ -10,7 +10,7 @@ RUN mkdir -p /home/dev WORKDIR /home/dev # Get mimalloc -RUN git clone https://github.com/microsoft/mimalloc -b dev-slice +RUN git clone https://github.com/microsoft/mimalloc -b dev2 RUN mkdir -p mimalloc/out/release RUN mkdir -p mimalloc/out/debug diff --git a/third-party/mimalloc/docker/manylinux-x64/Dockerfile b/third-party/mimalloc/docker/manylinux-x64/Dockerfile index 22d37e5a72..ff54d674ef 100644 --- a/third-party/mimalloc/docker/manylinux-x64/Dockerfile +++ b/third-party/mimalloc/docker/manylinux-x64/Dockerfile @@ -10,7 +10,7 @@ RUN mkdir -p /home/dev WORKDIR /home/dev # Get mimalloc -RUN git clone https://github.com/microsoft/mimalloc -b dev-slice +RUN git clone https://github.com/microsoft/mimalloc -b dev2 RUN mkdir -p mimalloc/out/release RUN mkdir -p mimalloc/out/debug diff --git a/third-party/mimalloc/ide/vs2017/mimalloc-override-test.vcxproj b/third-party/mimalloc/ide/vs2017/mimalloc-override-test.vcxproj deleted file mode 100644 index 04c16a9faf..0000000000 --- a/third-party/mimalloc/ide/vs2017/mimalloc-override-test.vcxproj +++ /dev/null @@ -1,190 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FEF7868F-750E-4C21-A04D-22707CC66879} - mimalloc-override-test - mimalloc-override-test - 10.0.19041.0 - - - - Application - true - v141 - - - Application - false - v141 - true - - - Application - true - v141 - - - Application - false - v141 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - MultiThreadedDebugDLL - false - Default - false - - - Console - kernel32.lib;%(AdditionalDependencies) - - - - - - - - - - Level3 - Disabled - true - true - ..\..\include - MultiThreadedDebugDLL - Sync - Default - false - - - Console - - - kernel32.lib;%(AdditionalDependencies) - - - - - - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - MultiThreadedDLL - - - true - true - Console - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - - - - - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - MultiThreadedDLL - - - true - true - Console - - - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - - - - - - - - - {abb5eae7-b3e6-432e-b636-333449892ea7} - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2017/mimalloc-override.vcxproj b/third-party/mimalloc/ide/vs2017/mimalloc-override.vcxproj deleted file mode 100644 index 6d20eb578f..0000000000 --- a/third-party/mimalloc/ide/vs2017/mimalloc-override.vcxproj +++ /dev/null @@ -1,260 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {ABB5EAE7-B3E6-432E-B636-333449892EA7} - mimalloc-override - mimalloc-override - 10.0.19041.0 - - - - DynamicLibrary - true - v141 - - - DynamicLibrary - false - v141 - - - DynamicLibrary - true - v141 - - - DynamicLibrary - false - v141 - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - - Level3 - Disabled - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); - MultiThreadedDebugDLL - false - Default - - - $(ProjectDir)\..\..\bin\mimalloc-redirect32.lib;%(AdditionalDependencies) - - - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect32.dll" "$(OutputPath)" - - - Copy mimalloc-redirect32.dll to the output directory - - - - - Level3 - Disabled - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); - MultiThreadedDebugDLL - false - Default - - - $(ProjectDir)\..\..\bin\mimalloc-redirect.lib;bcrypt.lib;%(AdditionalDependencies) - - - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect.dll" "$(OutputPath)" - - - copy mimalloc-redirect.dll to the output directory - - - - - Level3 - MaxSpeed - true - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - MultiThreadedDLL - Default - false - - - true - true - $(ProjectDir)\..\..\bin\mimalloc-redirect32.lib;%(AdditionalDependencies) - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect32.dll" "$(OutputPath)" - - - Copy mimalloc-redirect32.dll to the output directory - - - - - Level3 - MaxSpeed - true - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - MultiThreadedDLL - Default - false - - - true - true - $(ProjectDir)\..\..\bin\mimalloc-redirect.lib;bcrypt.lib;%(AdditionalDependencies) - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect.dll" "$(OutputPath)" - - - copy mimalloc-redirect.dll to the output directory - - - - - - - - - - - - - - - - false - false - false - false - - - true - true - true - true - - - - - - - - - - - - - true - true - true - true - - - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2017/mimalloc-test-stress.vcxproj b/third-party/mimalloc/ide/vs2017/mimalloc-test-stress.vcxproj deleted file mode 100644 index 061b8605c8..0000000000 --- a/third-party/mimalloc/ide/vs2017/mimalloc-test-stress.vcxproj +++ /dev/null @@ -1,159 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FEF7958F-750E-4C21-A04D-22707CC66878} - mimalloc-test-stress - mimalloc-test-stress - 10.0.19041.0 - - - - Application - true - v141 - - - Application - false - v141 - true - - - Application - true - v141 - - - Application - false - v141 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - - - Console - - - - - Level3 - Disabled - true - true - ..\..\include - - - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - %(PreprocessorDefinitions);NDEBUG - - - true - true - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - %(PreprocessorDefinitions);NDEBUG - - - true - true - Console - - - - - false - false - false - false - - - - - {abb5eae7-b3e6-432e-b636-333449892ea6} - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2017/mimalloc-test.vcxproj b/third-party/mimalloc/ide/vs2017/mimalloc-test.vcxproj deleted file mode 100644 index 04bd6537b4..0000000000 --- a/third-party/mimalloc/ide/vs2017/mimalloc-test.vcxproj +++ /dev/null @@ -1,158 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FEF7858F-750E-4C21-A04D-22707CC66878} - mimalloctest - mimalloc-test - 10.0.19041.0 - - - - Application - true - v141 - - - Application - false - v141 - true - - - Application - true - v141 - - - Application - false - v141 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - stdcpp17 - - - Console - - - - - Level3 - Disabled - true - true - ..\..\include - stdcpp14 - - - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - stdcpp17 - - - true - true - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - stdcpp17 - - - true - true - Console - - - - - {abb5eae7-b3e6-432e-b636-333449892ea6} - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2017/mimalloc.sln b/third-party/mimalloc/ide/vs2017/mimalloc.sln deleted file mode 100644 index 515c03f2e7..0000000000 --- a/third-party/mimalloc/ide/vs2017/mimalloc.sln +++ /dev/null @@ -1,71 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26228.102 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} - EndGlobalSection -EndGlobal diff --git a/third-party/mimalloc/ide/vs2017/mimalloc.vcxproj b/third-party/mimalloc/ide/vs2017/mimalloc.vcxproj deleted file mode 100644 index ece9a14d75..0000000000 --- a/third-party/mimalloc/ide/vs2017/mimalloc.vcxproj +++ /dev/null @@ -1,260 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {ABB5EAE7-B3E6-432E-B636-333449892EA6} - mimalloc - 10.0.19041.0 - mimalloc - - - - StaticLibrary - true - v141 - - - StaticLibrary - false - v141 - true - - - StaticLibrary - true - v141 - - - StaticLibrary - false - v141 - true - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - false - - - false - - - false - - - false - - - - Level3 - Disabled - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;MI_DEBUG=3;%(PreprocessorDefinitions); - CompileAsC - false - stdcpp17 - - - - - - - - - - - Level4 - Disabled - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;MI_DEBUG=3;%(PreprocessorDefinitions); - CompileAsCpp - false - stdcpp14 - - - - - - - - - - - - - - - - - - - Level3 - MaxSpeed - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - false - Default - CompileAsC - true - - - true - true - - - - - - - - - - - Level4 - MaxSpeed - true - true - ../../include - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - false - Default - CompileAsC - true - - - true - true - - - - - - - - - - - - - - - - - false - false - false - false - - - true - true - true - true - - - - - - - - - - - - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2019/mimalloc-override-test.vcxproj b/third-party/mimalloc/ide/vs2019/mimalloc-override-test.vcxproj deleted file mode 100644 index 7a9202f1b1..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc-override-test.vcxproj +++ /dev/null @@ -1,190 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FEF7868F-750E-4C21-A04D-22707CC66879} - mimalloc-override-test - 10.0 - mimalloc-override-test - - - - Application - true - v142 - - - Application - false - v142 - true - - - Application - true - v142 - - - Application - false - v142 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - MultiThreadedDebugDLL - Sync - Default - false - - - Console - kernel32.lib;%(AdditionalDependencies) - - - - - - - - - - Level3 - Disabled - true - true - ..\..\include - MultiThreadedDebugDLL - Sync - Default - false - - - Console - - - kernel32.lib;%(AdditionalDependencies) - - - - - - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - MultiThreadedDLL - - - true - true - Console - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - - - - - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - MultiThreadedDLL - - - true - true - Console - - - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - - - - - - - - - - - - {abb5eae7-b3e6-432e-b636-333449892ea7} - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2019/mimalloc-override.vcxproj b/third-party/mimalloc/ide/vs2019/mimalloc-override.vcxproj deleted file mode 100644 index a84a517858..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc-override.vcxproj +++ /dev/null @@ -1,260 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {ABB5EAE7-B3E6-432E-B636-333449892EA7} - mimalloc-override - 10.0 - mimalloc-override - - - - DynamicLibrary - true - v142 - - - DynamicLibrary - false - v142 - - - DynamicLibrary - true - v142 - - - DynamicLibrary - false - v142 - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .dll - mimalloc-override - - - - Level3 - Disabled - true - true - ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); - MultiThreadedDebugDLL - false - Default - - - $(ProjectDir)\..\..\bin\mimalloc-redirect32.lib;%(AdditionalDependencies) - - - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect32.dll" "$(OutputPath)" - - - Copy mimalloc-redirect32.dll to the output directory - - - - - Level3 - Disabled - true - true - ../../include - MI_DEBUG=3;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); - MultiThreadedDebugDLL - false - Default - - - $(ProjectDir)\..\..\bin\mimalloc-redirect.lib;%(AdditionalDependencies) - - - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect.dll" "$(OutputPath)" - - - copy mimalloc-redirect.dll to the output directory - - - - - Level3 - MaxSpeed - true - true - true - ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - MultiThreadedDLL - Default - false - - - true - true - $(ProjectDir)\..\..\bin\mimalloc-redirect32.lib;%(AdditionalDependencies) - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect32.dll" "$(OutputPath)" - - - Copy mimalloc-redirect32.dll to the output directory - - - - - Level3 - MaxSpeed - true - true - true - ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - MultiThreadedDLL - Default - false - - - true - true - $(ProjectDir)\..\..\bin\mimalloc-redirect.lib;%(AdditionalDependencies) - - - Default - false - - - COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect.dll" "$(OutputPath)" - - - copy mimalloc-redirect.dll to the output directory - - - - - - - - - - - - - - - - false - false - false - false - - - true - true - true - true - - - - - - - - - - - - - true - true - true - true - - - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2019/mimalloc-test-api.vcxproj b/third-party/mimalloc/ide/vs2019/mimalloc-test-api.vcxproj deleted file mode 100644 index 812a9cb116..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc-test-api.vcxproj +++ /dev/null @@ -1,155 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FFF7958F-750E-4C21-A04D-22707CC66878} - mimalloc-test-api - 10.0 - mimalloc-test-api - - - - Application - true - v142 - - - Application - false - v142 - true - - - Application - true - v142 - - - Application - false - v142 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - - - Console - - - - - Level3 - Disabled - true - true - ..\..\include - - - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - %(PreprocessorDefinitions);NDEBUG - - - true - true - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - %(PreprocessorDefinitions);NDEBUG - - - true - true - Console - - - - - - - - - {abb5eae7-b3e6-432e-b636-333449892ea6} - - - - - - diff --git a/third-party/mimalloc/ide/vs2019/mimalloc-test-stress.vcxproj b/third-party/mimalloc/ide/vs2019/mimalloc-test-stress.vcxproj deleted file mode 100644 index ef7ab3575a..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc-test-stress.vcxproj +++ /dev/null @@ -1,159 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FEF7958F-750E-4C21-A04D-22707CC66878} - mimalloc-test-stress - 10.0 - mimalloc-test-stress - - - - Application - true - v142 - - - Application - false - v142 - true - - - Application - true - v142 - - - Application - false - v142 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - - - Console - - - - - Level3 - Disabled - true - true - ..\..\include - - - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - %(PreprocessorDefinitions);NDEBUG - - - true - true - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - %(PreprocessorDefinitions);NDEBUG - - - true - true - Console - - - - - false - false - false - false - - - - - {abb5eae7-b3e6-432e-b636-333449892ea6} - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2019/mimalloc-test.vcxproj b/third-party/mimalloc/ide/vs2019/mimalloc-test.vcxproj deleted file mode 100644 index 13af6ab495..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc-test.vcxproj +++ /dev/null @@ -1,158 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {FEF7858F-750E-4C21-A04D-22707CC66878} - mimalloctest - 10.0 - mimalloc-test - - - - Application - true - v142 - - - Application - false - v142 - true - - - Application - true - v142 - - - Application - false - v142 - true - - - - - - - - - - - - - - - - - - - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - - - - Level3 - Disabled - true - true - ..\..\include - stdcpp17 - - - Console - - - - - Level3 - Disabled - true - true - ..\..\include - stdcpp17 - - - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - stdcpp17 - - - true - true - Console - - - - - Level3 - MaxSpeed - true - true - true - true - ..\..\include - _MBCS;%(PreprocessorDefinitions);NDEBUG - stdcpp17 - - - true - true - Console - - - - - {abb5eae7-b3e6-432e-b636-333449892ea6} - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2019/mimalloc.sln b/third-party/mimalloc/ide/vs2019/mimalloc.sln deleted file mode 100644 index 6ff01d3b44..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc.sln +++ /dev/null @@ -1,81 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29709.97 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-api", "mimalloc-test-api.vcxproj", "{FFF7958F-750E-4C21-A04D-22707CC66878}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} - EndGlobalSection -EndGlobal diff --git a/third-party/mimalloc/ide/vs2019/mimalloc.vcxproj b/third-party/mimalloc/ide/vs2019/mimalloc.vcxproj deleted file mode 100644 index 0076b1dbdd..0000000000 --- a/third-party/mimalloc/ide/vs2019/mimalloc.vcxproj +++ /dev/null @@ -1,258 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - 15.0 - {ABB5EAE7-B3E6-432E-B636-333449892EA6} - mimalloc - 10.0 - mimalloc - - - - StaticLibrary - true - v142 - - - StaticLibrary - false - v142 - true - - - StaticLibrary - true - v142 - - - StaticLibrary - false - v142 - true - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - .lib - mimalloc-static - - - - Level4 - Disabled - true - true - ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); - CompileAsCpp - false - Default - - - - - - - - - - - Level4 - Disabled - true - Default - ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); - CompileAsCpp - false - Default - - - - - - - - - - - - - - - - - - - Level4 - MaxSpeed - true - true - ../../include - %(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - false - Default - CompileAsCpp - true - Default - - - true - true - - - - - - - - - - - Level4 - MaxSpeed - true - true - ../../include - %(PreprocessorDefinitions);NDEBUG - AssemblyAndSourceCode - $(IntDir) - false - false - Default - CompileAsCpp - true - Default - - - true - true - - - - - - - - - - - - - - - - - false - false - false - false - - - true - true - true - true - - - - - - false - - - - - - - true - true - true - true - - - - true - true - true - true - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2022/mimalloc.vcxproj b/third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj similarity index 51% rename from third-party/mimalloc/ide/vs2022/mimalloc.vcxproj rename to third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj index 33ad9cef13..9009b99f1a 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc.vcxproj +++ b/third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj @@ -1,10 +1,26 @@  + + Debug + ARM64 + + + Debug + ARM64EC + Debug Win32 + + Release + ARM64 + + + Release + ARM64EC + Release Win32 @@ -21,9 +37,9 @@ 15.0 {ABB5EAE7-B3E6-432E-B636-333449892EA6} - mimalloc + mimalloc-lib 10.0 - mimalloc + mimalloc-lib @@ -42,12 +58,34 @@ true v143 + + StaticLibrary + true + v143 + + + StaticLibrary + true + v143 + StaticLibrary false v143 true + + StaticLibrary + false + v143 + true + + + StaticLibrary + false + v143 + true + @@ -62,9 +100,21 @@ + + + + + + + + + + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ @@ -84,12 +134,36 @@ .lib mimalloc-static + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + mimalloc-static + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + mimalloc-static + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .lib mimalloc-static + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + mimalloc-static + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + mimalloc-static + Level4 @@ -116,7 +190,61 @@ true Default ../../include - MI_DEBUG=4;MI_SECURE=0;%(PreprocessorDefinitions); + MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); + CompileAsCpp + false + stdcpp20 + + + + + + + + + + + + + + + + + + + Level4 + Disabled + true + Default + ../../include + MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); + CompileAsCpp + false + stdcpp20 + + + + + + + + + + + + + + + + + + + Level4 + Disabled + true + Default + ../../include + MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -198,30 +326,124 @@ + + + Level4 + MaxSpeed + true + Default + ../../include + %(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + false + Default + CompileAsCpp + true + stdcpp20 + CPUExtensionRequirementsARMv81 + Sync + + + true + true + + + + + + + + + + + + + + + + + Level4 + MaxSpeed + true + Default + ../../include + %(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + false + Default + CompileAsCpp + true + stdcpp20 + CPUExtensionRequirementsARMv81 + Sync + + + true + true + + + + + + + + + + + + + + false false + false + false false false + false + false true true true + true + true true + true + true + + true + true + true + true + true + true + false + false + false true + true + true true true true + true + true @@ -231,14 +453,22 @@ true true true + true + true true + true + true true true true + true + true true + true + true diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj.filters b/third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj.filters new file mode 100644 index 0000000000..90703da888 --- /dev/null +++ b/third-party/mimalloc/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -0,0 +1,105 @@ + + + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + + + {1430490c-e711-4ace-a1b8-36f4d5105873} + + + {461c78ef-04b0-44d1-a0ca-7d488abaa592} + + + \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-override.vcxproj b/third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj similarity index 51% rename from third-party/mimalloc/ide/vs2022/mimalloc-override.vcxproj rename to third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj index df2a081690..c1849bb261 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc-override.vcxproj +++ b/third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj @@ -1,10 +1,26 @@  + + Debug + ARM64 + + + Debug + ARM64EC + Debug Win32 + + Release + ARM64 + + + Release + ARM64EC + Release Win32 @@ -21,9 +37,9 @@ 15.0 {ABB5EAE7-B3E6-432E-B636-333449892EA7} - mimalloc-override + mimalloc-override-dll 10.0 - mimalloc-override + mimalloc-override-dll @@ -41,11 +57,31 @@ true v143 + + DynamicLibrary + true + v143 + + + DynamicLibrary + true + v143 + DynamicLibrary false v143 + + DynamicLibrary + false + v143 + + + DynamicLibrary + false + v143 + @@ -60,33 +96,69 @@ + + + + + + + + + + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc-override + mimalloc $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc-override + mimalloc $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc-override + mimalloc + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + mimalloc + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + mimalloc $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc-override + mimalloc + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + mimalloc + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + mimalloc @@ -98,7 +170,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp $(ProjectDir)\..\..\bin\mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -126,7 +198,7 @@ MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp $(ProjectDir)\..\..\bin\mimalloc-redirect.lib;%(AdditionalDependencies) @@ -144,6 +216,62 @@ copy mimalloc-redirect.dll to the output directory + + + Level3 + Disabled + true + true + ../../include + MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MultiThreadedDebugDLL + false + CompileAsCpp + + + $(ProjectDir)\..\..\bin\mimalloc-redirect-arm64.lib;%(AdditionalDependencies) + + + + + Default + false + + + COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect-arm64.dll" "$(OutputPath)" + + + copy mimalloc-redirect-arm64.dll to the output directory + + + + + Level3 + Disabled + true + true + ../../include + MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MultiThreadedDebugDLL + false + CompileAsCpp + + + $(ProjectDir)\..\..\bin\mimalloc-redirect-arm64ec.lib;%(AdditionalDependencies) + + + + + Default + false + + + COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect-arm64ec.dll" "$(OutputPath)" + + + copy mimalloc-redirect-arm64ec.dll to the output directory + + Level3 @@ -157,7 +285,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp false @@ -189,7 +317,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp false @@ -208,6 +336,72 @@ copy mimalloc-redirect.dll to the output directory + + + Level3 + MaxSpeed + true + true + true + ../../include + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + MultiThreadedDLL + CompileAsCpp + false + CPUExtensionRequirementsARMv81 + + + true + true + $(ProjectDir)\..\..\bin\mimalloc-redirect-arm64.lib;%(AdditionalDependencies) + + + Default + false + + + COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect-arm64.dll" "$(OutputPath)" + + + copy mimalloc-redirect-arm64.dll to the output directory + + + + + Level3 + MaxSpeed + true + true + true + ../../include + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + MultiThreadedDLL + CompileAsCpp + false + CPUExtensionRequirementsARMv81 + + + true + true + $(ProjectDir)\..\..\bin\mimalloc-redirect-arm64ec.lib;%(AdditionalDependencies) + + + Default + false + + + COPY /Y "$(ProjectDir)..\..\bin\mimalloc-redirect-arm64ec.dll" "$(OutputPath)" + + + copy mimalloc-redirect-arm64ec.dll to the output directory + + @@ -226,16 +420,34 @@ false false false + false + false false + false + false true true true + true + true true + true + true + + true + true + true + true + true + true + true + true + @@ -246,7 +458,11 @@ true true true + true + true true + true + true @@ -254,7 +470,11 @@ true true true + true + true true + true + true diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj.filters b/third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj.filters new file mode 100644 index 0000000000..91bdf95c73 --- /dev/null +++ b/third-party/mimalloc/ide/vs2022/mimalloc-override-dll.vcxproj.filters @@ -0,0 +1,113 @@ + + + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + Sources + + + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + Headers + + + + + {262c6c21-e270-4ba6-bd63-4ac999307e4e} + + + {94b40bdc-a741-45dd-81aa-c05fabcd2970} + + + + + Sources + + + \ No newline at end of file diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-override-test.vcxproj b/third-party/mimalloc/ide/vs2022/mimalloc-override-test.vcxproj index a3c56f7bad..427a75ae14 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc-override-test.vcxproj +++ b/third-party/mimalloc/ide/vs2022/mimalloc-override-test.vcxproj @@ -1,10 +1,26 @@ + + Debug + ARM64 + + + Debug + ARM64EC + Debug Win32 + + Release + ARM64 + + + Release + ARM64EC + Release Win32 @@ -23,7 +39,7 @@ {FEF7868F-750E-4C21-A04D-22707CC66879} mimalloc-override-test 10.0 - mimalloc-override-test + mimalloc-test-override @@ -42,12 +58,34 @@ true v143 + + Application + true + v143 + + + Application + true + v143 + Application false v143 true + + Application + false + v143 + true + + + Application + false + v143 + true + @@ -62,9 +100,21 @@ + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ @@ -78,10 +128,26 @@ $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + Level3 @@ -128,6 +194,54 @@ + + + Level3 + Disabled + true + true + ..\..\include + MultiThreadedDebugDLL + Sync + Default + false + + + Console + + + kernel32.lib;%(AdditionalDependencies) + + + + + + + + + + Level3 + Disabled + true + true + ..\..\include + MultiThreadedDebugDLL + Sync + Default + false + + + Console + + + kernel32.lib;%(AdditionalDependencies) + + + + + + + Level3 @@ -176,11 +290,61 @@ + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + MultiThreadedDLL + + + true + true + Console + + + kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + MultiThreadedDLL + + + true + true + Console + + + kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + - + {abb5eae7-b3e6-432e-b636-333449892ea7} diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-test-api.vcxproj b/third-party/mimalloc/ide/vs2022/mimalloc-test-api.vcxproj index d9b9cae4f1..b7f97ad204 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc-test-api.vcxproj +++ b/third-party/mimalloc/ide/vs2022/mimalloc-test-api.vcxproj @@ -1,10 +1,26 @@ + + Debug + ARM64 + + + Debug + ARM64EC + Debug Win32 + + Release + ARM64 + + + Release + ARM64EC + Release Win32 @@ -42,12 +58,34 @@ true v143 + + Application + true + v143 + + + Application + true + v143 + Application false v143 true + + Application + false + v143 + true + + + Application + false + v143 + true + @@ -62,9 +100,21 @@ + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ @@ -78,10 +128,26 @@ $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + Level3 @@ -106,6 +172,30 @@ Console + + + Level3 + Disabled + true + true + ..\..\include + + + Console + + + + + Level3 + Disabled + true + true + ..\..\include + + + Console + + Level3 @@ -140,19 +230,59 @@ Console + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + %(PreprocessorDefinitions);NDEBUG + + + true + true + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + %(PreprocessorDefinitions);NDEBUG + + + true + true + Console + + true true true + true + true true + true + true false + false + false - + {abb5eae7-b3e6-432e-b636-333449892ea6} diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-test-stress.vcxproj b/third-party/mimalloc/ide/vs2022/mimalloc-test-stress.vcxproj index 14bd3e6927..cb761f9429 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc-test-stress.vcxproj +++ b/third-party/mimalloc/ide/vs2022/mimalloc-test-stress.vcxproj @@ -1,10 +1,26 @@ + + Debug + ARM64 + + + Debug + ARM64EC + Debug Win32 + + Release + ARM64 + + + Release + ARM64EC + Release Win32 @@ -42,12 +58,34 @@ true v143 + + Application + true + v143 + + + Application + true + v143 + Application false v143 true + + Application + false + v143 + true + + + Application + false + v143 + true + @@ -62,9 +100,21 @@ + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ @@ -78,10 +128,26 @@ $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + Level3 @@ -106,6 +172,30 @@ Console + + + Level3 + Disabled + true + true + ..\..\include + + + Console + + + + + Level3 + Disabled + true + true + ..\..\include + + + Console + + Level3 @@ -140,17 +230,57 @@ Console + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + %(PreprocessorDefinitions);NDEBUG + CPUExtensionRequirementsARMv81 + + + true + true + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + %(PreprocessorDefinitions);NDEBUG + CPUExtensionRequirementsARMv81 + + + true + true + Console + + false false + false + false false false + false + false - - {abb5eae7-b3e6-432e-b636-333449892ea7} + + {abb5eae7-b3e6-432e-b636-333449892ea6} diff --git a/third-party/mimalloc/ide/vs2022/mimalloc-test.vcxproj b/third-party/mimalloc/ide/vs2022/mimalloc-test.vcxproj index 506dd7d457..83202dbed6 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc-test.vcxproj +++ b/third-party/mimalloc/ide/vs2022/mimalloc-test.vcxproj @@ -1,10 +1,26 @@ + + Debug + ARM64 + + + Debug + ARM64EC + Debug Win32 + + Release + ARM64 + + + Release + ARM64EC + Release Win32 @@ -23,7 +39,7 @@ {FEF7858F-750E-4C21-A04D-22707CC66878} mimalloctest 10.0 - mimalloc-test + mimalloc-test-static @@ -42,12 +58,34 @@ true v143 + + Application + true + v143 + + + Application + true + v143 + Application false v143 true + + Application + false + v143 + true + + + Application + false + v143 + true + @@ -62,9 +100,21 @@ + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ @@ -78,10 +128,26 @@ $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + Level3 @@ -108,6 +174,32 @@ Console + + + Level3 + Disabled + true + true + ..\..\include + stdcpp17 + + + Console + + + + + Level3 + Disabled + true + true + ..\..\include + stdcpp17 + + + Console + + Level3 @@ -144,13 +236,49 @@ Console + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + stdcpp17 + + + true + true + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + stdcpp17 + + + true + true + Console + + - - {abb5eae7-b3e6-432e-b636-333449892ea6} - + - + + {abb5eae7-b3e6-432e-b636-333449892ea6} + diff --git a/third-party/mimalloc/ide/vs2022/mimalloc.sln b/third-party/mimalloc/ide/vs2022/mimalloc.sln index 6ff01d3b44..040af3aca4 100644 --- a/third-party/mimalloc/ide/vs2022/mimalloc.sln +++ b/third-party/mimalloc/ide/vs2022/mimalloc.sln @@ -1,81 +1,133 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29709.97 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-api", "mimalloc-test-api.vcxproj", "{FFF7958F-750E-4C21-A04D-22707CC66878}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 - {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 - {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35527.113 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-lib", "mimalloc-lib.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-dll", "mimalloc-override-dll.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-api", "mimalloc-test-api.vcxproj", "{FFF7958F-750E-4C21-A04D-22707CC66878}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM64 = Debug|ARM64 + Debug|ARM64EC = Debug|ARM64EC + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|ARM64 = Release|ARM64 + Release|ARM64EC = Release|ARM64EC + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|ARM64.Build.0 = Debug|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|ARM64.ActiveCfg = Release|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|ARM64.Build.0 = Release|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|ARM64EC.Build.0 = Release|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|ARM64.Build.0 = Debug|ARM64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|ARM64.ActiveCfg = Release|ARM64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|ARM64.Build.0 = Release|ARM64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|ARM64EC.Build.0 = Release|ARM64EC + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|ARM64.Build.0 = Debug|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|ARM64.ActiveCfg = Release|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|ARM64.Build.0 = Release|ARM64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|ARM64EC.Build.0 = Release|ARM64EC + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|ARM64.Build.0 = Debug|ARM64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|ARM64.ActiveCfg = Release|ARM64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|ARM64.Build.0 = Release|ARM64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|ARM64EC.Build.0 = Release|ARM64EC + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64.Build.0 = Debug|ARM64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64.ActiveCfg = Release|ARM64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64.Build.0 = Release|ARM64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64EC.Build.0 = Release|ARM64EC + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64.Build.0 = Debug|ARM64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64EC.ActiveCfg = Debug|ARM64EC + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|ARM64EC.Build.0 = Debug|ARM64EC + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64.ActiveCfg = Release|ARM64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64.Build.0 = Release|ARM64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64EC.ActiveCfg = Release|ARM64EC + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|ARM64EC.Build.0 = Release|ARM64EC + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FFF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} + EndGlobalSection +EndGlobal diff --git a/third-party/mimalloc/include/mimalloc.h b/third-party/mimalloc/include/mimalloc.h index c41bcc8039..bd91db4338 100644 --- a/third-party/mimalloc/include/mimalloc.h +++ b/third-party/mimalloc/include/mimalloc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +Copyright (c) 2018-2025, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 217 // major + 2 digits minor +#define MI_MALLOC_VERSION 219 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes @@ -149,6 +149,7 @@ typedef void (mi_cdecl mi_error_fun)(int err, void* arg); mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); mi_decl_export void mi_collect(bool force) mi_attr_noexcept; +mi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept; mi_decl_export int mi_version(void) mi_attr_noexcept; mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; @@ -259,11 +260,12 @@ typedef struct mi_heap_area_s { size_t used; // number of allocated blocks size_t block_size; // size in bytes of each block size_t full_block_size; // size in bytes of a full block including padding and metadata. + int heap_tag; // heap tag associated with this area } mi_heap_area_t; typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); -mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); +mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); // Experimental mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; @@ -275,7 +277,7 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; -mi_decl_export void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept; +mi_decl_export void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept; // Experimental: heaps associated with specific memory arena's typedef int mi_arena_id_t; @@ -289,8 +291,31 @@ mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_co mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); #endif + +// Experimental: allow sub-processes whose memory segments stay separated (and no reclamation between them) +// Used for example for separate interpreter's in one process. +typedef void* mi_subproc_id_t; +mi_decl_export mi_subproc_id_t mi_subproc_main(void); +mi_decl_export mi_subproc_id_t mi_subproc_new(void); +mi_decl_export void mi_subproc_delete(mi_subproc_id_t subproc); +mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet) + +// Experimental: visit abandoned heap areas (from threads that have been terminated) +mi_decl_export bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); + +// Experimental: create a new heap with a specified heap tag. Set `allow_destroy` to false to allow the thread +// to reclaim abandoned memory (with a compatible heap_tag and arena_id) but in that case `mi_heap_destroy` will +// fall back to `mi_heap_delete`. +mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id); + // deprecated -mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; +mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; + +// Experimental: objects followed by a guard page. +// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object. +// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages. +mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed); +mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max); // ------------------------------------------------------ @@ -332,7 +357,7 @@ typedef enum mi_option_e { mi_option_deprecated_segment_cache, mi_option_deprecated_page_reset, mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination - mi_option_deprecated_segment_reset, + mi_option_deprecated_segment_reset, mi_option_eager_commit_delay, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10) mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. @@ -348,6 +373,13 @@ typedef enum mi_option_e { mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1) mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) + mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) + mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0) + mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) + mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) + mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) + mi_option_target_segments_per_thread, // experimental (=0) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/third-party/mimalloc/include/mimalloc/atomic.h b/third-party/mimalloc/include/mimalloc/atomic.h index d5333dd90f..0d7aaf789b 100644 --- a/third-party/mimalloc/include/mimalloc/atomic.h +++ b/third-party/mimalloc/include/mimalloc/atomic.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023 Microsoft Research, Daan Leijen +Copyright (c) 2018-2024 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -8,10 +8,21 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_ATOMIC_H #define MIMALLOC_ATOMIC_H +// include windows.h or pthreads.h +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)) +#define MI_USE_PTHREADS +#include +#endif + // -------------------------------------------------------------------------------------------- // Atomics // We need to be portable between C, C++, and MSVC. -// We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. +// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode. // This is why we try to use only `uintptr_t` and `*` as atomic types. // To gain better insight in the range of used atomics, we use explicitly named memory order operations // instead of passing the memory order as a parameter. @@ -24,9 +35,9 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_atomic(name) std::atomic_##name #define mi_memory_order(name) std::memory_order_##name #if (__cplusplus >= 202002L) // c++20, see issue #571 -#define MI_ATOMIC_VAR_INIT(x) x + #define MI_ATOMIC_VAR_INIT(x) x #elif !defined(ATOMIC_VAR_INIT) -#define MI_ATOMIC_VAR_INIT(x) x + #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) #endif @@ -61,6 +72,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) #define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) #define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed)) #define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) #define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) #define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) @@ -99,6 +111,7 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des) #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des) #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x) #define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x) #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x) #else @@ -107,6 +120,7 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des) #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des) #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x) #define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x) #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x) #endif @@ -133,10 +147,6 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { #elif defined(_MSC_VER) // Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include #include #ifdef _WIN64 typedef LONG64 msc_intptr_t; @@ -302,11 +312,16 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { return (intptr_t)mi_atomic_addi(p, -sub); } + +// ---------------------------------------------------------------------- +// Once and Guard +// ---------------------------------------------------------------------- + typedef _Atomic(uintptr_t) mi_atomic_once_t; // Returns true only on the first invocation static inline bool mi_atomic_once( mi_atomic_once_t* once ) { - if (mi_atomic_load_relaxed(once) != 0) return false; // quick test + if (mi_atomic_load_relaxed(once) != 0) return false; // quick test uintptr_t expected = 0; return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1 } @@ -322,17 +337,16 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t; +// ---------------------------------------------------------------------- // Yield +// ---------------------------------------------------------------------- + #if defined(__cplusplus) #include static inline void mi_atomic_yield(void) { std::this_thread::yield(); } #elif defined(_WIN32) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include static inline void mi_atomic_yield(void) { YieldProcessor(); } @@ -390,4 +404,134 @@ static inline void mi_atomic_yield(void) { #endif +// ---------------------------------------------------------------------- +// Locks +// These do not have to be recursive and should be light-weight +// in-process only locks. Only used for reserving arena's and to +// maintain the abandoned list. +// ---------------------------------------------------------------------- +#if _MSC_VER +#pragma warning(disable:26110) // unlock with holding lock +#endif + +#define mi_lock(lock) for(bool _go = (mi_lock_acquire(lock),true); _go; (mi_lock_release(lock), _go=false) ) + +#if defined(_WIN32) + +#if 1 +#define mi_lock_t SRWLOCK // slim reader-writer lock + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return TryAcquireSRWLockExclusive(lock); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + AcquireSRWLockExclusive(lock); +} +static inline void mi_lock_release(mi_lock_t* lock) { + ReleaseSRWLockExclusive(lock); +} +static inline void mi_lock_init(mi_lock_t* lock) { + InitializeSRWLock(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} + +#else +#define mi_lock_t CRITICAL_SECTION + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return TryEnterCriticalSection(lock); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + EnterCriticalSection(lock); +} +static inline void mi_lock_release(mi_lock_t* lock) { + LeaveCriticalSection(lock); +} +static inline void mi_lock_init(mi_lock_t* lock) { + InitializeCriticalSection(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + DeleteCriticalSection(lock); +} + +#endif + +#elif defined(MI_USE_PTHREADS) + +void _mi_error_message(int err, const char* fmt, ...); + +#define mi_lock_t pthread_mutex_t + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return (pthread_mutex_trylock(lock) == 0); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + const int err = pthread_mutex_lock(lock); + if (err != 0) { + _mi_error_message(err, "internal error: lock cannot be acquired\n"); + } +} +static inline void mi_lock_release(mi_lock_t* lock) { + pthread_mutex_unlock(lock); +} +static inline void mi_lock_init(mi_lock_t* lock) { + pthread_mutex_init(lock, NULL); +} +static inline void mi_lock_done(mi_lock_t* lock) { + pthread_mutex_destroy(lock); +} + +#elif defined(__cplusplus) + +#include +#define mi_lock_t std::mutex + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + return lock->try_lock(); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + lock->lock(); +} +static inline void mi_lock_release(mi_lock_t* lock) { + lock->unlock(); +} +static inline void mi_lock_init(mi_lock_t* lock) { + (void)(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} + +#else + +// fall back to poor man's locks. +// this should only be the case in a single-threaded environment (like __wasi__) + +#define mi_lock_t _Atomic(uintptr_t) + +static inline bool mi_lock_try_acquire(mi_lock_t* lock) { + uintptr_t expected = 0; + return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1); +} +static inline void mi_lock_acquire(mi_lock_t* lock) { + for (int i = 0; i < 1000; i++) { // for at most 1000 tries? + if (mi_lock_try_acquire(lock)) return; + mi_atomic_yield(); + } +} +static inline void mi_lock_release(mi_lock_t* lock) { + mi_atomic_store_release(lock, (uintptr_t)0); +} +static inline void mi_lock_init(mi_lock_t* lock) { + mi_lock_release(lock); +} +static inline void mi_lock_done(mi_lock_t* lock) { + (void)(lock); +} + +#endif + + #endif // __MIMALLOC_ATOMIC_H diff --git a/third-party/mimalloc/include/mimalloc/internal.h b/third-party/mimalloc/include/mimalloc/internal.h index 6c6e5ed04f..e7e7b50835 100644 --- a/third-party/mimalloc/include/mimalloc/internal.h +++ b/third-party/mimalloc/include/mimalloc/internal.h @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------- -// This file contains the interal API's of mimalloc and various utility +// This file contains the internal API's of mimalloc and various utility // functions and macros. // -------------------------------------------------------------------------- @@ -31,16 +31,25 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) #define mi_decl_weak +#define mi_decl_hidden #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) #define mi_decl_weak __attribute__((weak)) +#define mi_decl_hidden __attribute__((visibility("hidden"))) +#elif __cplusplus >= 201103L // c++11 +#define mi_decl_noinline +#define mi_decl_thread thread_local +#define mi_decl_cache_align alignas(MI_CACHE_LINE) +#define mi_decl_weak +#define mi_decl_hidden #else #define mi_decl_noinline #define mi_decl_thread __thread // hope for the best :-) #define mi_decl_cache_align #define mi_decl_weak +#define mi_decl_hidden #endif #if defined(__EMSCRIPTEN__) && !defined(__wasi__) @@ -53,150 +62,175 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_externc #endif -// pthreads -#if !defined(_WIN32) && !defined(__wasi__) -#define MI_USE_PTHREADS -#include -#endif +// "libc.c" +#include +void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args); +void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); +char _mi_toupper(char c); +int _mi_strnicmp(const char* s, const char* t, size_t n); +void _mi_strlcpy(char* dest, const char* src, size_t dest_size); +void _mi_strlcat(char* dest, const char* src, size_t dest_size); +size_t _mi_strlen(const char* s); +size_t _mi_strnlen(const char* s, size_t max_len); +bool _mi_getenv(const char* name, char* result, size_t result_size); // "options.c" -void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); -void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); -void _mi_warning_message(const char* fmt, ...); -void _mi_verbose_message(const char* fmt, ...); -void _mi_trace_message(const char* fmt, ...); -void _mi_options_init(void); -void _mi_error_message(int err, const char* fmt, ...); +void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); +void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); +void _mi_warning_message(const char* fmt, ...); +void _mi_verbose_message(const char* fmt, ...); +void _mi_trace_message(const char* fmt, ...); +void _mi_options_init(void); +long _mi_option_get_fast(mi_option_t option); +void _mi_error_message(int err, const char* fmt, ...); // random.c -void _mi_random_init(mi_random_ctx_t* ctx); -void _mi_random_init_weak(mi_random_ctx_t* ctx); -void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); -void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); -uintptr_t _mi_random_next(mi_random_ctx_t* ctx); -uintptr_t _mi_heap_random_next(mi_heap_t* heap); -uintptr_t _mi_os_random_weak(uintptr_t extra_seed); +void _mi_random_init(mi_random_ctx_t* ctx); +void _mi_random_init_weak(mi_random_ctx_t* ctx); +void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); +void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); +uintptr_t _mi_random_next(mi_random_ctx_t* ctx); +uintptr_t _mi_heap_random_next(mi_heap_t* heap); +uintptr_t _mi_os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c extern mi_decl_cache_align mi_stats_t _mi_stats_main; -extern mi_decl_cache_align const mi_page_t _mi_page_empty; -bool _mi_is_main_thread(void); -size_t _mi_current_thread_count(void); -bool _mi_preloading(void); // true while the C runtime is not initialized yet +extern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty; +void _mi_process_load(void); +void mi_cdecl _mi_process_done(void); +bool _mi_is_redirected(void); +bool _mi_allocator_init(const char** message); +void _mi_allocator_done(void); +bool _mi_is_main_thread(void); +size_t _mi_current_thread_count(void); +bool _mi_preloading(void); // true while the C runtime is not initialized yet +void _mi_thread_done(mi_heap_t* heap); +void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap -void _mi_thread_done(mi_heap_t* heap); -void _mi_thread_data_collect(void); -void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_heap_guarded_init(mi_heap_t* heap); // os.c -void _mi_os_init(void); // called from process init -void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats); -void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats); -void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats); - -size_t _mi_os_page_size(void); -size_t _mi_os_good_alloc_size(size_t size); -bool _mi_os_has_overcommit(void); -bool _mi_os_has_virtual_reserve(void); - -bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); -bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); -bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); -bool _mi_os_protect(void* addr, size_t size); -bool _mi_os_unprotect(void* addr, size_t size); -bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats); - -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats); -void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats); - -void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); -bool _mi_os_use_large_page(size_t size, size_t alignment); -size_t _mi_os_large_page_size(void); - -void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); +void _mi_os_init(void); // called from process init +void* _mi_os_alloc(size_t size, mi_memid_t* memid); +void _mi_os_free(void* p, size_t size, mi_memid_t memid); +void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid); + +size_t _mi_os_page_size(void); +size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); +bool _mi_os_has_virtual_reserve(void); + +bool _mi_os_reset(void* addr, size_t size); +bool _mi_os_commit(void* p, size_t size, bool* is_zero); +bool _mi_os_decommit(void* addr, size_t size); +bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_purge(void* p, size_t size); +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size); + +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); + +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); +bool _mi_os_use_large_page(size_t size, size_t alignment); +size_t _mi_os_large_page_size(void); + +void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); // arena.c mi_arena_id_t _mi_arena_id_none(void); -void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats); -void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); -void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); -bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); -bool _mi_arena_contains(const void* p); -void _mi_arenas_collect(bool force_purge, mi_stats_t* stats); -void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); - -bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment); -void _mi_arena_segment_mark_abandoned(mi_segment_t* segment); -size_t _mi_arena_segment_abandoned_count(void); - -typedef struct mi_arena_field_cursor_s { // abstract - mi_arena_id_t start; - int count; - size_t bitmap_idx; +void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid); +void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid); +bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); +bool _mi_arena_contains(const void* p); +void _mi_arenas_collect(bool force_purge); +void _mi_arena_unsafe_destroy_all(void); + +bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment); +void _mi_arena_segment_mark_abandoned(mi_segment_t* segment); + +void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid); +void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size); + +typedef struct mi_arena_field_cursor_s { // abstract struct + size_t os_list_count; // max entries to visit in the OS abandoned list + size_t start; // start arena idx (may need to be wrapped) + size_t end; // end arena idx (exclusive, may need to be wrapped) + size_t bitmap_idx; // current bit idx for an arena + mi_subproc_t* subproc; // only visit blocks in this sub-process + bool visit_all; // ensure all abandoned blocks are seen (blocking) + bool hold_visit_lock; // if the subproc->abandoned_os_visit_lock is held } mi_arena_field_cursor_t; -void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current); +void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool visit_all, mi_arena_field_cursor_t* current); mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous); +void _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current); // "segment-map.c" -void _mi_segment_map_allocated_at(const mi_segment_t* segment); -void _mi_segment_map_freed_at(const mi_segment_t* segment); +void _mi_segment_map_allocated_at(const mi_segment_t* segment); +void _mi_segment_map_freed_at(const mi_segment_t* segment); +void _mi_segment_map_unsafe_destroy(void); // "segment.c" -mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); +mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld); void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); -void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld); +void _mi_segment_collect(mi_segment_t* segment, bool force); #if MI_HUGE_PAGE_ABANDON -void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); +void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); #else -void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); +void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); #endif uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); -void _mi_abandoned_await_readers(void); void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment); +bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); // "page.c" -void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; + +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks +void _mi_page_unfull(mi_page_t* page); +void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page +void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... +void _mi_page_force_abandon(mi_page_t* page); -void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks -void _mi_page_unfull(mi_page_t* page); -void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page -void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... -void _mi_heap_delayed_free_all(mi_heap_t* heap); -bool _mi_heap_delayed_free_partial(mi_heap_t* heap); -void _mi_heap_collect_retired(mi_heap_t* heap, bool force); +void _mi_heap_delayed_free_all(mi_heap_t* heap); +bool _mi_heap_delayed_free_partial(mi_heap_t* heap); +void _mi_heap_collect_retired(mi_heap_t* heap, bool force); -void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); -bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); -size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); -void _mi_deferred_free(mi_heap_t* heap, bool force); +void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); +size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); +void _mi_deferred_free(mi_heap_t* heap, bool force); -void _mi_page_free_collect(mi_page_t* page,bool force); -void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments +void _mi_page_free_collect(mi_page_t* page,bool force); +void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments -size_t _mi_bin_size(uint8_t bin); // for stats -uint8_t _mi_bin(size_t size); // for stats +size_t _mi_bin_size(uint8_t bin); // for stats +uint8_t _mi_bin(size_t size); // for stats // "heap.c" -void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag); -void _mi_heap_destroy_pages(mi_heap_t* heap); -void _mi_heap_collect_abandon(mi_heap_t* heap); -void _mi_heap_set_default_direct(mi_heap_t* heap); -bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); -void _mi_heap_unsafe_destroy_all(void); -mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag); +void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag); +void _mi_heap_destroy_pages(mi_heap_t* heap); +void _mi_heap_collect_abandon(mi_heap_t* heap); +void _mi_heap_set_default_direct(mi_heap_t* heap); +bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); +void _mi_heap_unsafe_destroy_all(mi_heap_t* heap); +mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag); +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); // "stats.c" -void _mi_stats_done(mi_stats_t* stats); +void _mi_stats_done(mi_stats_t* stats); mi_msecs_t _mi_clock_now(void); mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); @@ -213,18 +247,6 @@ bool _mi_free_delayed_block(mi_block_t* block); void _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); -// "libc.c" -#include -void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args); -void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); -char _mi_toupper(char c); -int _mi_strnicmp(const char* s, const char* t, size_t n); -void _mi_strlcpy(char* dest, const char* src, size_t dest_size); -void _mi_strlcat(char* dest, const char* src, size_t dest_size); -size_t _mi_strlen(const char* s); -size_t _mi_strnlen(const char* s, size_t max_len); -bool _mi_getenv(const char* name, char* result, size_t result_size); - #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); #endif @@ -349,6 +371,14 @@ static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { return (divider == 0 ? size : ((size + divider - 1) / divider)); } + +// clamp an integer +static inline size_t _mi_clamp(size_t sz, size_t min, size_t max) { + if (sz < min) return min; + else if (sz > max) return max; + else return sz; +} + // Is memory zero initialized? static inline bool mi_mem_is_zero(const void* p, size_t size) { for (size_t i = 0; i < size; i++) { @@ -410,7 +440,7 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot Heap functions ------------------------------------------------------------------------------------------- */ -extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap +extern mi_decl_hidden const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap static inline bool mi_heap_is_backing(const mi_heap_t* heap) { return (heap->tld->heap_backing == heap); @@ -418,11 +448,11 @@ static inline bool mi_heap_is_backing(const mi_heap_t* heap) { static inline bool mi_heap_is_initialized(mi_heap_t* heap) { mi_assert_internal(heap != NULL); - return (heap != &_mi_heap_empty); + return (heap != NULL && heap != &_mi_heap_empty); } static inline uintptr_t _mi_ptr_cookie(const void* p) { - extern mi_heap_t _mi_heap_main; + extern mi_decl_hidden mi_heap_t _mi_heap_main; mi_assert_internal(_mi_heap_main.cookie != 0); return ((uintptr_t)p ^ _mi_heap_main.cookie); } @@ -589,7 +619,7 @@ static inline bool mi_page_immediate_available(const mi_page_t* page) { } // is more than 7/8th of a page in use? -static inline bool mi_page_mostly_used(const mi_page_t* page) { +static inline bool mi_page_is_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; return (page->reserved - page->used <= frac); @@ -620,6 +650,39 @@ static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { page->flags.x.has_aligned = has_aligned; } +/* ------------------------------------------------------------------- + Guarded objects +------------------------------------------------------------------- */ +#if MI_GUARDED +static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED); +} + +static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) { + // this code is written to result in fast assembly as it is on the hot path for allocation + const size_t count = heap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time.. + if mi_likely(count != 0) { + // no sample + heap->guarded_sample_count = count; + return false; + } + else if (size >= heap->guarded_size_min && size <= heap->guarded_size_max) { + // use guarded allocation + heap->guarded_sample_count = heap->guarded_sample_rate; // reset + return (heap->guarded_sample_rate != 0); + } + else { + // failed size criteria, rewind count (but don't write to an empty heap) + if (heap->guarded_sample_rate != 0) { heap->guarded_sample_count = 1; } + return false; + } +} + +mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; + +#endif + /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers @@ -679,6 +742,16 @@ static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; } +static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { + const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys)); + // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951) + #ifdef MI_BIG_ENDIAN + return (x & 0x00FFFFFF); + #else + return (x & 0xFFFFFF00); + #endif +} + static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; @@ -803,7 +876,7 @@ static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool static inline uintptr_t _mi_random_shuffle(uintptr_t x) { if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros -#if (MI_INTPTR_SIZE==8) +#if (MI_INTPTR_SIZE>=8) // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; @@ -825,13 +898,13 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { // Optimize numa node access for the common case (= one node) // ------------------------------------------------------------------- -int _mi_os_numa_node_get(mi_os_tld_t* tld); +int _mi_os_numa_node_get(void); size_t _mi_os_numa_node_count_get(void); -extern _Atomic(size_t) _mi_numa_node_count; -static inline int _mi_os_numa_node(mi_os_tld_t* tld) { +extern mi_decl_hidden _Atomic(size_t) _mi_numa_node_count; +static inline int _mi_os_numa_node(void) { if mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1) { return 0; } - else return _mi_os_numa_node_get(tld); + else return _mi_os_numa_node_get(); } static inline size_t _mi_os_numa_node_count(void) { const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count); @@ -893,16 +966,18 @@ static inline size_t mi_ctz(uintptr_t x) { } #else -static inline size_t mi_ctz32(uint32_t x) { + +static inline size_t mi_ctz_generic32(uint32_t x) { // de Bruijn multiplication, see - static const unsigned char debruijn[32] = { + static const uint8_t debruijn[32] = { 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; if (x==0) return 32; - return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27]; + return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27]; } -static inline size_t mi_clz32(uint32_t x) { + +static inline size_t mi_clz_generic32(uint32_t x) { // de Bruijn multiplication, see static const uint8_t debruijn[32] = { 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1, @@ -914,28 +989,37 @@ static inline size_t mi_clz32(uint32_t x) { x |= x >> 4; x |= x >> 8; x |= x >> 16; - return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27]; + return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27]; } -static inline size_t mi_clz(uintptr_t x) { - if (x==0) return MI_INTPTR_BITS; -#if (MI_INTPTR_BITS <= 32) - return mi_clz32((uint32_t)x); -#else - size_t count = mi_clz32((uint32_t)(x >> 32)); - if (count < 32) return count; - return (32 + mi_clz32((uint32_t)x)); -#endif +static inline size_t mi_ctz(size_t x) { + if (x==0) return MI_SIZE_BITS; + #if (MI_SIZE_BITS <= 32) + return mi_ctz_generic32((uint32_t)x); + #else + const uint32_t lo = (uint32_t)x; + if (lo != 0) { + return mi_ctz_generic32(lo); + } + else { + return (32 + mi_ctz_generic32((uint32_t)(x>>32))); + } + #endif } -static inline size_t mi_ctz(uintptr_t x) { - if (x==0) return MI_INTPTR_BITS; -#if (MI_INTPTR_BITS <= 32) - return mi_ctz32((uint32_t)x); -#else - size_t count = mi_ctz32((uint32_t)x); - if (count < 32) return count; - return (32 + mi_ctz32((uint32_t)(x>>32))); -#endif + +static inline size_t mi_clz(size_t x) { + if (x==0) return MI_SIZE_BITS; + #if (MI_SIZE_BITS <= 32) + return mi_clz_generic32((uint32_t)x); + #else + const uint32_t hi = (uint32_t)(x>>32); + if (hi != 0) { + return mi_clz_generic32(hi); + } + else { + return 32 + mi_clz_generic32((uint32_t)x); + } + #endif } #endif @@ -957,8 +1041,9 @@ static inline size_t mi_bsr(uintptr_t x) { #if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) #include extern bool _mi_cpu_has_fsrm; +extern bool _mi_cpu_has_erms; static inline void _mi_memcpy(void* dst, const void* src, size_t n) { - if (_mi_cpu_has_fsrm) { + if ((_mi_cpu_has_fsrm && n <= 128) || (_mi_cpu_has_erms && n > 128)) { __movsb((unsigned char*)dst, (const unsigned char*)src, n); } else { @@ -966,7 +1051,7 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) { } } static inline void _mi_memzero(void* dst, size_t n) { - if (_mi_cpu_has_fsrm) { + if ((_mi_cpu_has_fsrm && n <= 128) || (_mi_cpu_has_erms && n > 128)) { __stosb((unsigned char*)dst, 0, n); } else { diff --git a/third-party/mimalloc/include/mimalloc/prim.h b/third-party/mimalloc/include/mimalloc/prim.h index 3f4574ddd9..606b719943 100644 --- a/third-party/mimalloc/include/mimalloc/prim.h +++ b/third-party/mimalloc/include/mimalloc/prim.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +Copyright (c) 2018-2024, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -25,6 +25,8 @@ typedef struct mi_os_mem_config_s { size_t page_size; // default to 4KiB size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows) size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) + size_t physical_memory; // physical memory size + size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization) bool has_overcommit; // can we reserve more memory than can be actually committed? bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc) bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) @@ -41,9 +43,10 @@ int _mi_prim_free(void* addr, size_t size ); // If `commit` is false, the virtual memory range only needs to be reserved (with no access) // which will later be committed explicitly using `_mi_prim_commit`. // `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored. // pre: !commit => !allow_large // try_alignment >= _mi_os_page_size() and a power of 2 -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); // Commit memory. Returns error code or 0 on success. // For example, on Linux this would make the memory PROT_READ|PROT_WRITE. @@ -115,14 +118,14 @@ void _mi_prim_thread_done_auto_done(void); void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); + + + //------------------------------------------------------------------- -// Thread id: `_mi_prim_thread_id()` -// -// Getting the thread id should be performant as it is called in the -// fast path of `_mi_free` and we specialize for various platforms as -// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. -// We only require _mi_prim_thread_id() to return a unique id -// for each thread (unequal to zero). +// Access to TLS (thread local storage) slots. +// We need fast access to both a unique thread id (in `free.c:mi_free`) and +// to a thread-local heap pointer (in `alloc.c:mi_malloc`). +// To achieve this we use specialized code for various platforms. //------------------------------------------------------------------- // On some libc + platform combinations we can directly access a thread-local storage (TLS) slot. @@ -134,14 +137,14 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); // but unfortunately we can not detect support reliably (see issue #883) // We also use it on Apple OS as we use a TLS slot for the default heap there. #if defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \ - || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ ) -#define MI_HAS_TLS_SLOT +#define MI_HAS_TLS_SLOT 1 static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { void* res; @@ -202,13 +205,58 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce #endif } +#elif _WIN32 && MI_WIN_USE_FIXED_TLS && !defined(MI_WIN_USE_FLS) + +// On windows we can store the thread-local heap at a fixed TLS slot to avoid +// thread-local initialization checks in the fast path. This uses a fixed location +// in the TCB though (last user-reserved slot by default) which may clash with other applications. + +#define MI_HAS_TLS_SLOT 2 // 2 = we can reliably initialize the slot (saving a test on each malloc) + +#if MI_WIN_USE_FIXED_TLS > 1 +#define MI_TLS_SLOT (MI_WIN_USE_FIXED_TLS) +#elif MI_SIZE_SIZE == 4 +#define MI_TLS_SLOT (0x710) // Last user-reserved slot +// #define MI_TLS_SLOT (0xF0C) // Last TlsSlot (might clash with other app reserved slot) +#else +#define MI_TLS_SLOT (0x888) // Last user-reserved slot +// #define MI_TLS_SLOT (0x1678) // Last TlsSlot (might clash with other app reserved slot) +#endif + +static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { + #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) + return (void*)__readgsqword((unsigned long)slot); // direct load at offset from gs + #elif _M_IX86 && !defined(_M_ARM64EC) + return (void*)__readfsdword((unsigned long)slot); // direct load at offset from fs + #else + return ((void**)NtCurrentTeb())[slot / sizeof(void*)]; + #endif +} +static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { + ((void**)NtCurrentTeb())[slot / sizeof(void*)] = value; +} + #endif + + +//------------------------------------------------------------------- +// Get a fast unique thread id. +// +// Getting the thread id should be performant as it is called in the +// fast path of `_mi_free` and we specialize for various platforms as +// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. +// We only require _mi_prim_thread_id() to return a unique id +// for each thread (unequal to zero). +//------------------------------------------------------------------- + + // Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id // but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) // Nevertheless, it seems needed on older graviton platforms (see issue #851). // For now, we only enable this for specific platforms. #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \ + && !defined(__CYGWIN__) \ && !defined(MI_LIBC_MUSL) \ && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */ #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ @@ -235,10 +283,6 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { #elif defined(_WIN32) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { // Windows: works on Intel and ARM in both 32- and 64-bit return (uintptr_t)NtCurrentTeb(); @@ -251,7 +295,7 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { return (uintptr_t)__builtin_thread_pointer(); } -#elif defined(MI_HAS_TLS_SLOT) +#elif MI_HAS_TLS_SLOT static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { #if defined(__BIONIC__) @@ -278,7 +322,8 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { /* ---------------------------------------------------------------------------------------- -The thread local default heap: `_mi_prim_get_default_heap()` +Get the thread local default heap: `_mi_prim_get_default_heap()` + This is inlined here as it is on the fast path for allocation functions. On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a @@ -315,19 +360,21 @@ static inline mi_heap_t* mi_prim_get_default_heap(void); #endif -#if defined(MI_TLS_SLOT) +#if MI_TLS_SLOT # if !defined(MI_HAS_TLS_SLOT) # error "trying to use a TLS slot for the default heap, but the mi_prim_tls_slot primitives are not defined" # endif static inline mi_heap_t* mi_prim_get_default_heap(void) { mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT); + #if MI_HAS_TLS_SLOT == 1 // check if the TLS slot is initialized if mi_unlikely(heap == NULL) { #ifdef __GNUC__ __asm(""); // prevent conditional load of the address of _mi_heap_empty #endif heap = (mi_heap_t*)&_mi_heap_empty; } + #endif return heap; } @@ -369,5 +416,4 @@ static inline mi_heap_t* mi_prim_get_default_heap(void) { #endif // mi_prim_get_default_heap() - #endif // MIMALLOC_PRIM_H diff --git a/third-party/mimalloc/include/mimalloc/track.h b/third-party/mimalloc/include/mimalloc/track.h index a659d94044..4b5709e2b5 100644 --- a/third-party/mimalloc/include/mimalloc/track.h +++ b/third-party/mimalloc/include/mimalloc/track.h @@ -34,7 +34,7 @@ The corresponding `mi_track_free` still uses the block start pointer and origina The `mi_track_resize` is currently unused but could be called on reallocations within a block. `mi_track_init` is called at program start. -The following macros are for tools like asan and valgrind to track whether memory is +The following macros are for tools like asan and valgrind to track whether memory is defined, undefined, or not accessible at all: #define mi_track_mem_defined(p,size) @@ -82,10 +82,6 @@ defined, undefined, or not accessible at all: #define MI_TRACK_HEAP_DESTROY 1 #define MI_TRACK_TOOL "ETW" -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include #include "../src/prim/windows/etw.h" #define mi_track_init() EventRegistermicrosoft_windows_mimalloc(); @@ -96,7 +92,7 @@ defined, undefined, or not accessible at all: // no tracking #define MI_TRACK_ENABLED 0 -#define MI_TRACK_HEAP_DESTROY 0 +#define MI_TRACK_HEAP_DESTROY 0 #define MI_TRACK_TOOL "none" #define mi_track_malloc_size(p,reqsize,size,zero) diff --git a/third-party/mimalloc/include/mimalloc/types.h b/third-party/mimalloc/include/mimalloc/types.h index 2fdde904bb..df6ff26c4f 100644 --- a/third-party/mimalloc/include/mimalloc/types.h +++ b/third-party/mimalloc/include/mimalloc/types.h @@ -73,6 +73,13 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif +// Use guard pages behind objects of a certain size (set by the MIMALLOC_DEBUG_GUARDED_MIN/MAX options) +// Padding should be disabled when using guard pages +// #define MI_GUARDED 1 +#if defined(MI_GUARDED) +#define MI_PADDING 0 +#endif + // Reserve extra padding at the end of each block to be more resilient against heap block overflows. // The padding can detect buffer overflow on free. #if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) @@ -193,7 +200,7 @@ typedef int32_t mi_ssize_t; #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 8KiB on 64-bit #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit #define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE) -#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit +#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 16MiB on 64-bit #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) // Maximum number of size classes. (spaced exponentially in 12.5% increments) @@ -237,13 +244,20 @@ typedef struct mi_block_s { mi_encoded_t next; } mi_block_t; +#if MI_GUARDED +// we always align guarded pointers in a block at an offset +// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones +#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) +#endif + // The delayed flags are used for efficient multi-threaded free-ing typedef enum mi_delayed_e { MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list - MI_NEVER_DELAYED_FREE = 3 // sticky: used for abondoned pages without a owning heap; this only resets on page reclaim + MI_NEVER_DELAYED_FREE = 3 // sticky: used for abandoned pages without a owning heap; this only resets on page reclaim } mi_delayed_t; @@ -260,7 +274,7 @@ typedef union mi_page_flags_s { #else // under thread sanitizer, use a byte for each flag to suppress warning, issue #130 typedef union mi_page_flags_s { - uint16_t full_aligned; + uint32_t full_aligned; struct { uint8_t in_full; uint8_t has_aligned; @@ -319,7 +333,7 @@ typedef struct mi_page_s { mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) uint16_t used; // number of blocks in use (including blocks in `thread_free`) uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`) - uint8_t heap_tag; // tag of the owning heap, used for separated heaps by object type + uint8_t heap_tag; // tag of the owning heap, used to separate heaps by object type // padding size_t block_size; // size available in each block (always `>0`) uint8_t* page_start; // start of the page area containing the blocks @@ -408,7 +422,7 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) { typedef struct mi_memid_os_info { void* base; // actual base address of the block (used for offset aligned allocations) - size_t alignment; // alignment at allocation + size_t size; // full allocation size } mi_memid_os_info_t; typedef struct mi_memid_arena_info { @@ -430,7 +444,7 @@ typedef struct mi_memid_s { // ----------------------------------------------------------------------------------------- -// Segments are large allocated memory blocks (8mb on 64 bit) from arenas or the OS. +// Segments are large allocated memory blocks (32mb on 64 bit) from arenas or the OS. // // Inside segments we allocated fixed size mimalloc pages (`mi_page_t`) that contain blocks. // The start of a segment is this structure with a fixed number of slice entries (`slices`) @@ -442,12 +456,16 @@ typedef struct mi_memid_s { // For slices, the `block_size` field is repurposed to signify if a slice is used (`1`) or not (`0`). // Small and medium pages use a fixed amount of slices to reduce slice fragmentation, while // large and huge pages span a variable amount of slices. + +typedef struct mi_subproc_s mi_subproc_t; + typedef struct mi_segment_s { // constant fields mi_memid_t memid; // memory id for arena/OS allocation bool allow_decommit; // can we decommmit the memory bool allow_purge; // can we purge the memory (reset or decommit) size_t segment_size; + mi_subproc_t* subproc; // segment belongs to sub process // segment fields mi_msecs_t purge_expire; // purge slices in the `purge_mask` after this time @@ -457,12 +475,16 @@ typedef struct mi_segment_s { // from here is zero initialized struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) bool was_reclaimed; // true if it was reclaimed (used to limit on-free reclamation) + bool dont_free; // can be temporarily true to ensure the segment is not freed size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) size_t abandoned_visits; // count how often this segment is visited during abondoned reclamation (to force reclaim if it takes too long) size_t used; // count of pages in use uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` + struct mi_segment_s* abandoned_os_next; // only used for abandoned segments outside arena's, and only if `mi_option_visit_abandoned` is enabled + struct mi_segment_s* abandoned_os_prev; + size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT` size_t segment_info_slices; // initial count of slices that we are using for segment info and possible guard pages. @@ -540,6 +562,13 @@ struct mi_heap_s { mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom tag, can be used for separating heaps based on the object types + #if MI_GUARDED + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) + size_t guarded_sample_seed; // starting sample count + size_t guarded_sample_count; // current sample count (counting down to 0) + #endif mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") }; @@ -633,24 +662,34 @@ typedef struct mi_stats_s { mi_stat_counter_t arena_count; mi_stat_counter_t arena_crossover_count; mi_stat_counter_t arena_rollback_count; + mi_stat_counter_t guarded_alloc_count; #if MI_STAT>1 mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; #endif } mi_stats_t; +// add to stat keeping track of the peak void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); +// adjust stat in special cases to compensate for double counting +void _mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount); +void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount); +// counters can just be increased void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); #if (MI_STAT) #define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) #define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) #define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) +#define mi_stat_adjust_increase(stat,amount) _mi_stat_adjust_increase( &(stat), amount) +#define mi_stat_adjust_decrease(stat,amount) _mi_stat_adjust_decrease( &(stat), amount) #else -#define mi_stat_increase(stat,amount) (void)0 -#define mi_stat_decrease(stat,amount) (void)0 -#define mi_stat_counter_increase(stat,amount) (void)0 +#define mi_stat_increase(stat,amount) ((void)0) +#define mi_stat_decrease(stat,amount) ((void)0) +#define mi_stat_counter_increase(stat,amount) ((void)0) +#define mi_stat_adjuct_increase(stat,amount) ((void)0) +#define mi_stat_adjust_decrease(stat,amount) ((void)0) #endif #define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) @@ -658,6 +697,21 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); #define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) +// ------------------------------------------------------ +// Sub processes do not reclaim or visit segments +// from other sub processes +// ------------------------------------------------------ + +struct mi_subproc_s { + _Atomic(size_t) abandoned_count; // count of abandoned segments for this sub-process + _Atomic(size_t) abandoned_os_list_count; // count of abandoned segments in the os-list + mi_lock_t abandoned_os_lock; // lock for the abandoned os segment list (outside of arena's) (this lock protect list operations) + mi_lock_t abandoned_os_visit_lock; // ensure only one thread per subproc visits the abandoned os list + mi_segment_t* abandoned_os_list; // doubly-linked list of abandoned segments outside of arena's (in OS allocated memory) + mi_segment_t* abandoned_os_list_tail; // the tail-end of the list + mi_memid_t memid; // provenance of this memory block +}; + // ------------------------------------------------------ // Thread Local data // ------------------------------------------------------ @@ -672,13 +726,6 @@ typedef struct mi_span_queue_s { #define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT) -// OS thread local data -typedef struct mi_os_tld_s { - size_t region_idx; // start point for next allocation - mi_stats_t* stats; // points to tld stats -} mi_os_tld_t; - - // Segments thread local data typedef struct mi_segments_tld_s { mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments @@ -687,8 +734,8 @@ typedef struct mi_segments_tld_s { size_t current_size; // current size of all segments size_t peak_size; // peak size of all segments size_t reclaim_count;// number of reclaimed (abandoned) segments + mi_subproc_t* subproc; // sub-process this thread belongs to. mi_stats_t* stats; // points to tld stats - mi_os_tld_t* os; // points to os stats } mi_segments_tld_t; // Thread local data @@ -698,7 +745,6 @@ struct mi_tld_s { mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) mi_segments_tld_t segments; // segment tld - mi_os_tld_t os; // os tld mi_stats_t stats; // statistics }; diff --git a/third-party/mimalloc/readme.md b/third-party/mimalloc/readme.md index a0296b43c3..264da2b56e 100644 --- a/third-party/mimalloc/readme.md +++ b/third-party/mimalloc/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.1.7` (2024-05-21). -Latest v1 tag: `v1.8.7` (2024-05-21). +Latest release tag: `v2.1.9` (2025-01-03). +Latest v1 tag: `v1.8.9` (2024-01-03). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -22,7 +22,7 @@ without code changes, for example, on dynamically linked ELF-based systems (Linu ``` It also includes a robust way to override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: -- __small and consistent__: the library is about 8k LOC using simple and +- __small and consistent__: the library is about 10k LOC using simple and consistent data structures. This makes it very suitable to integrate and adapt in other projects. For runtime systems it provides hooks for a monotonic _heartbeat_ and deferred freeing (for @@ -70,35 +70,40 @@ Enjoy! ### Branches -* `master`: latest stable release (based on `dev-slice`). +* `master`: latest stable release (based on `dev2`). * `dev`: development branch for mimalloc v1. Use this branch for submitting PR's. -* `dev-slice`: development branch for mimalloc v2. This branch is downstream of `dev` (and is essentially equal to `dev` except for -`src/segment.c`) +* `dev2`: development branch for mimalloc v2. This branch is downstream of `dev` + (and is essentially equal to `dev` except for `src/segment.c`). Uses larger sliced segments to manage + mimalloc pages what can reduce fragmentation. +* `dev3`: development branch for mimalloc v3-alpha. This branch is downstream of `dev`. This is still experimental, + but simplifies previous versions by having no segments any more. This improves sharing of memory + between threads, and on certain large workloads uses less memory with less fragmentation. ### Releases -Note: the `v2.x` version has a different algorithm for managing internal mimalloc pages (as slices) that tends to use reduce -memory usage - and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance - (see [below](#performance)); please report if you observe any significant performance regression. - +* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS + guard pages behind objects to catch buffer overflows as they occur. + Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for + thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; + consistent mimalloc.dll on Windows (instead of mimalloc-override.dll); fix mimalloc-redirect on Win11 H2; + add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. * 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches. * 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation. * 2024-04-22, `v1.8.4`, `v2.1.4`: Fixes various bugs and build issues. Add `MI_LIBC_MUSL` cmake flag for musl builds. Free-ing code is refactored into a separate module (`free.c`). Mimalloc page info is simplified with the block size - directly available (and new `block_size_shift` to improve aligned block free-ing). + directly available (and new `block_size_shift` to improve aligned block free-ing). New approach to collection of abandoned segments: When a thread terminates the segments it owns are abandoned (containing still live objects) and these can be - reclaimed by other threads. We no longer use a list of abandoned segments but this is now done using bitmaps in arena's + reclaimed by other threads. We no longer use a list of abandoned segments but this is now done using bitmaps in arena's which is more concurrent (and more aggressive). Abandoned memory can now also be reclaimed if a thread frees an object in an abandoned page (which can be disabled using `mi_option_abandoned_reclaim_on_free`). The option `mi_option_max_segment_reclaim` gives a maximum percentage of abandoned segments that can be reclaimed per try (=10%). -* 2023-04-24, `v1.8.2`, `v2.1.2`: Fixes build issues on freeBSD, musl, and C17 (UE 5.1.1). Reduce code size/complexity +* 2023-04-24, `v1.8.2`, `v2.1.2`: Fixes build issues on freeBSD, musl, and C17 (UE 5.1.1). Reduce code size/complexity by removing regions and segment-cache's and only use arenas with improved memory purging -- this may improve memory usage as well for larger services. Renamed options for consistency. Improved Valgrind and ASAN checking. - + * 2023-04-03, `v1.8.1`, `v2.1.1`: Fixes build issues on some platforms. * 2023-03-29, `v1.8.0`, `v2.1.0`: Improved support dynamic overriding on Windows 11. Improved tracing precision @@ -106,14 +111,14 @@ memory usage abstraction layer to make it easier to port and separate platform dependent code (in `src/prim`). Fixed C++ STL compilation on older Microsoft C++ compilers, and various small bug fixes. * 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with [asan](#asan) and improved [Valgrind](#valgrind) support. - Support arbitrary large alignments (in particular for `std::pmr` pools). - Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). - Heap walks now visit all object (including huge objects). Support Windows nano server containers (by Johannes Schindelin,@dscho). + Support arbitrary large alignments (in particular for `std::pmr` pools). + Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). + Heap walks now visit all object (including huge objects). Support Windows nano server containers (by Johannes Schindelin,@dscho). Various small bug fixes. * 2022-11-03, `v1.7.7`, `v2.0.7`: Initial support for [Valgrind](#valgrind) for leak testing and heap block overflow detection. Initial - support for attaching heaps to a speficic memory area (only in v2). Fix `realloc` behavior for zero size blocks, remove restriction to integral multiple of the alignment in `alloc_align`, improved aligned allocation performance, reduced contention with many threads on few processors (thank you @dposluns!), vs2022 support, support `pkg-config`, . + support for attaching heaps to a specific memory area (only in v2). Fix `realloc` behavior for zero size blocks, remove restriction to integral multiple of the alignment in `alloc_align`, improved aligned allocation performance, reduced contention with many threads on few processors (thank you @dposluns!), vs2022 support, support `pkg-config`, . * 2022-04-14, `v1.7.6`, `v2.0.6`: fix fallback path for aligned OS allocation on Windows, improve Windows aligned allocation even when compiling with older SDK's, fix dynamic overriding on macOS Monterey, fix MSVC C++ dynamic overriding, fix @@ -164,9 +169,9 @@ The `mimalloc` project builds a static library (in `out/msvc-x64`), while the `mimalloc-override` project builds a DLL for overriding malloc in the entire program. -## macOS, Linux, BSD, etc. +## Linux, macOS, BSD, etc. -We use [`cmake`](https://cmake.org)1 as the build system: +We use [`cmake`](https://cmake.org) as the build system: ``` > mkdir -p out/release @@ -189,24 +194,42 @@ maintains detailed statistics as: > cmake -DCMAKE_BUILD_TYPE=Debug ../.. > make ``` + This will name the shared library as `libmimalloc-debug.so`. -Finally, you can build a _secure_ version that uses guard pages, encrypted -free lists, etc., as: +Finally, you can build a _secure_ version that uses guard pages, encrypted free lists, etc., as: + ``` > mkdir -p out/secure > cd out/secure > cmake -DMI_SECURE=ON ../.. > make ``` + This will name the shared library as `libmimalloc-secure.so`. -Use `ccmake`2 instead of `cmake` -to see and customize all the available build options. +Use `cmake ../.. -LH` to see all the available build options. + +The examples use the default compiler. If you like to use another, use: + +``` +> CC=clang CXX=clang++ cmake ../.. +``` -Notes: -1. Install CMake: `sudo apt-get install cmake` -2. Install CCMake: `sudo apt-get install cmake-curses-gui` +## Cmake with Visual Studio +You can also use cmake on Windows. Open a Visual Studio 2022 development prompt +and invoke `cmake` with the right [generator](https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html) +and architecture, like: + +``` +> cmake ..\.. -G "Visual Studio 17 2022" -A x64 -DMI_OVERRIDE=ON +``` + +The cmake build type is specified when actually building, for example: + +``` +> cmake --build . --config=Release +``` ## Single source @@ -226,7 +249,7 @@ mimalloc uses only safe OS calls (`mmap` and `VirtualAlloc`) and can co-exist with other allocators linked to the same program. If you use `cmake`, you can simply use: ``` -find_package(mimalloc 1.4 REQUIRED) +find_package(mimalloc 1.8 REQUIRED) ``` in your `CMakeLists.txt` to find a locally installed mimalloc. Then use either: ``` @@ -240,7 +263,7 @@ to link with the static library. See `test\CMakeLists.txt` for an example. For best performance in C++ programs, it is also recommended to override the global `new` and `delete` operators. For convenience, mimalloc provides -[`mimalloc-new-delete.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) which does this for you -- just include it in a single(!) source file in your project. +[`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) which does this for you -- just include it in a single(!) source file in your project. In C++, mimalloc also provides the `mi_stl_allocator` struct which implements the `std::allocator` interface. @@ -295,14 +318,14 @@ You can set further options either programmatically (using [`mi_option_set`](htt Advanced options: -- `MIMALLOC_ARENA_EAGER_COMMIT=2`: turns on eager commit for the large arenas (usually 1GiB) from which mimalloc - allocates segments and pages. Set this to 2 (default) to - only enable this on overcommit systems (e.g. Linux). Set this to 1 to enable explicitly on other systems - as well (like Windows or macOS) which may improve performance (as the whole arena is committed at once). - Note that eager commit only increases the commit but not the actual the peak resident set +- `MIMALLOC_ARENA_EAGER_COMMIT=2`: turns on eager commit for the large arenas (usually 1GiB) from which mimalloc + allocates segments and pages. Set this to 2 (default) to + only enable this on overcommit systems (e.g. Linux). Set this to 1 to enable explicitly on other systems + as well (like Windows or macOS) which may improve performance (as the whole arena is committed at once). + Note that eager commit only increases the commit but not the actual the peak resident set (rss) so it is generally ok to enable this. -- `MIMALLOC_PURGE_DELAY=N`: the delay in `N` milli-seconds (by default `10`) after which mimalloc will purge - OS pages that are not in use. This signals to the OS that the underlying physical memory can be reused which +- `MIMALLOC_PURGE_DELAY=N`: the delay in `N` milli-seconds (by default `10`) after which mimalloc will purge + OS pages that are not in use. This signals to the OS that the underlying physical memory can be reused which can reduce memory fragmentation especially in long running (server) programs. Setting `N` to `0` purges immediately when a page becomes unused which can improve memory usage but also decreases performance. Setting `N` to a higher value like `100` can improve performance (sometimes by a lot) at the cost of potentially using more memory at times. @@ -310,7 +333,7 @@ Advanced options: - `MIMALLOC_PURGE_DECOMMITS=1`: By default "purging" memory means unused memory is decommitted (`MEM_DECOMMIT` on Windows, `MADV_DONTNEED` (which decresease rss immediately) on `mmap` systems). Set this to 0 to instead "reset" unused memory on a purge (`MEM_RESET` on Windows, generally `MADV_FREE` (which does not decrease rss immediately) on `mmap` systems). - Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual + Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual address ranges and decommits within those ranges (to make the underlying physical memory available to other processes). Further options for large workloads and services: @@ -319,15 +342,16 @@ Further options for large workloads and services: at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA nodes (but this can happen in any case as NUMA local allocation is always a best effort but not guaranteed). -- `MIMALLOC_ALLOW_LARGE_OS_PAGES=1`: use large OS pages (2 or 4MiB) when available; for some workloads this can significantly - improve performance. When this option is disabled, it also disables transparent huge pages (THP) for the process - (on Linux and Android). Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs +- `MIMALLOC_ALLOW_LARGE_OS_PAGES=0`: Set to 1 to use large OS pages (2 or 4MiB) when available; for some workloads this can significantly + improve performance. When this option is disabled (default), it also disables transparent huge pages (THP) for the process + (on Linux and Android). On Linux the default setting is 2 -- this enables the use of large pages through THP only. + Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs to explicitly give permissions for large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that - can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead whenever possible). + can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead whenever possible). - `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where `N` is the number of 1GiB _huge_ OS pages. This reserves the huge pages at startup and sometimes this can give a large (latency) performance improvement on big workloads. - Usually it is better to not use `MIMALLOC_ALLOW_LARGE_OS_PAGES=1` in combination with this setting. Just like large + Usually it is better to not use `MIMALLOC_ALLOW_LARGE_OS_PAGES=1` in combination with this setting. Just like large OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at startup only once). @@ -367,13 +391,39 @@ As always, evaluate with care as part of an overall security strategy as all of ## Debug Mode -When _mimalloc_ is built using debug mode, various checks are done at runtime to catch development errors. +When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`), +various checks are done at runtime to catch development errors. - Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. - All objects have padding at the end to detect (byte precise) heap block overflows. - Double free's, and freeing invalid heap pointers are detected. - Corrupted free-lists and some forms of use-after-free are detected. +## Guarded Mode + +_mimalloc_ can be build in guarded mode using the `-DMI_GUARDED=ON` flags in `cmake`. +This enables placing OS guard pages behind certain object allocations to catch buffer overflows as they occur. +This can be invaluable to catch buffer-overflow bugs in large programs. However, it also means that any object +allocated with a guard page takes at least 8 KiB memory for the guard page and its alignment. As such, allocating +a guard page for every allocation may be too expensive both in terms of memory, and in terms of performance with +many system calls. Therefore, there are various environment variables (and options) to tune this: + +- `MIMALLOC_GUARDED_SAMPLE_RATE=N`: Set the sample rate to `N` (by default 4000). This mode places a guard page + behind every `N` suitable object allocations (per thread). Since the performance in guarded mode without placing + guard pages is close to release mode, this can be used to enable guard pages even in production to catch latent + buffer overflow bugs. Set the sample rate to `1` to guard every object, and to `0` to place no guard pages at all. + +- `MIMALLOC_GUARDED_SAMPLE_SEED=N`: Start sampling at `N` (by default random). Can be used to reproduce a buffer + overflow if needed. + +- `MIMALLOC_GUARDED_MIN=N`, `MIMALLOC_GUARDED_MAX=N`: Minimal and maximal _rounded_ object sizes for which a guard + page is considered (`0` and `1GiB` respectively). If you suspect a buffer overflow occurs with an object of size + 141, set the minimum and maximum to `148` and the sample rate to `1` to have all of those guarded. + +- `MIMALLOC_GUARDED_PRECISE=1`: If we have an object of size 13, we would usually place it an aligned 16 bytes in + front of the guard page. Using `MIMALLOC_GUARDED_PRECISE` places it exactly 13 bytes before a page so that even + a 1 byte overflow is detected. This violates the C/C++ minimal alignment guarantees though so use with care. + # Overriding Standard Malloc @@ -415,43 +465,48 @@ Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -### Dynamic Override on Windows +# Windows Override Dynamically overriding on mimalloc on Windows -is robust and has the particular advantage to be able to redirect all malloc/free calls that go through -the (dynamic) C runtime allocator, including those from other DLL's or libraries. -As it intercepts all allocation calls on a low level, it can be used reliably +is robust and has the particular advantage to be able to redirect all malloc/free calls +that go through the (dynamic) C runtime allocator, including those from other DLL's or +libraries. As it intercepts all allocation calls on a low level, it can be used reliably on large programs that include other 3rd party components. -There are four requirements to make the overriding work robustly: +There are four requirements to make the overriding work well: 1. Use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -2. Link your program explicitly with `mimalloc-override.dll` library. - To ensure the `mimalloc-override.dll` is loaded at run-time it is easiest to insert some - call to the mimalloc API in the `main` function, like `mi_version()` - (or use the `/INCLUDE:mi_version` switch on the linker). See the `mimalloc-override-test` project - for an example on how to use this. -3. The [`mimalloc-redirect.dll`](bin) (or `mimalloc-redirect32.dll`) must be put - in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency of that DLL). - The redirection DLL ensures that all calls to the C runtime malloc API get redirected to - mimalloc functions (which reside in `mimalloc-override.dll`). -4. Ensure the `mimalloc-override.dll` comes as early as possible in the import + +2. Link your program explicitly with the `mimalloc.lib` export library for the `mimalloc.dll`. + (which must be compiled with `-DMI_OVERRIDE=ON`, which is the default though). + To ensure the `mimalloc.dll` is actually loaded at run-time it is easiest + to insert some call to the mimalloc API in the `main` function, like `mi_version()` + (or use the `/include:mi_version` switch on the linker command, or + similarly, `#pragma comment(linker, "/include:mi_version")` in some source file). + See the `mimalloc-test-override` project for an example on how to use this. + +3. The `mimalloc-redirect.dll` must be put in the same directory as the main + `mimalloc.dll` at runtime (as it is a dependency of that DLL). + The redirection DLL ensures that all calls to the C runtime malloc API get + redirected to mimalloc functions (which reside in `mimalloc.dll`). + +4. Ensure the `mimalloc.dll` comes as early as possible in the import list of the final executable (so it can intercept all potential allocations). + You can use `minject -l ` to check this if needed. For best performance on Windows with C++, it is also recommended to also override the `new`/`delete` operations (by including -[`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) +[`mimalloc-new-delete.h`](include/mimalloc-new-delete.h) a single(!) source file in your project). The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic -overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. +overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully +redirected. + +For different platforms than x64, you may need a specific [redirection dll](bin). +Furthermore, we cannot always re-link an executable or ensure `mimalloc.dll` comes +first in the import table. In such cases the [`minject`](bin) tool can be used +to patch the executable's import tables. -We cannot always re-link an executable with `mimalloc-override.dll`, and similarly, we cannot always -ensure the the DLL comes first in the import table of the final executable. -In many cases though we can patch existing executables without any recompilation -if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the `mimalloc-override.dll` -into the import table (and put `mimalloc-redirect.dll` in the same folder) -Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388) or -the [`minject`](bin) program. ## Static override @@ -462,6 +517,7 @@ an object file instead of a library file as linkers give preference to that over archives to resolve symbols. To ensure that the standard malloc interface resolves to the _mimalloc_ library, link it as the first object file. For example: + ``` > gcc -o myprogram mimalloc.o myfile1.c ... ``` @@ -469,16 +525,17 @@ object file. For example: Another way to override statically that works on all platforms, is to link statically to mimalloc (as shown in the introduction) and include a header file in each source file that re-defines `malloc` etc. to `mi_malloc`. -This is provided by [`mimalloc-override.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-override.h). This only works reliably though if all sources are +This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works +reliably though if all sources are under your control or otherwise mixing of pointers from different heaps may occur! # Tools Generally, we recommend using the standard allocator with memory tracking tools, but mimalloc -can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. +can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. Moreover, it can be build to support Windows event tracing ([ETW]). -This has a small performance overhead but does allow detecting memory leaks and byte-precise +This has a small performance overhead but does allow detecting memory leaks and byte-precise buffer overflows directly on final executables. See also the `test/test-wrong.c` file to test with various tools. ## Valgrind @@ -505,9 +562,13 @@ you also need to tell `valgrind` to not intercept those calls itself, and use: By setting the `MIMALLOC_SHOW_STATS` environment variable you can check that mimalloc is indeed used and not the standard allocator. Even though the [Valgrind option][valgrind-soname] -is called `--soname-synonyms`, this also -works when overriding with a static library or object file. Unfortunately, it is not possible to -dynamically override mimalloc using `LD_PRELOAD` together with `valgrind`. +is called `--soname-synonyms`, this also works when overriding with a static library or object file. +To dynamically override mimalloc using `LD_PRELOAD` together with `valgrind`, use: + +``` +> valgrind --trace-children=yes --soname-synonyms=somalloc=*mimalloc* /usr/bin/env LD_PRELOAD=/usr/lib/libmimalloc.so -- +``` + See also the `test/test-wrong.c` file to test with `valgrind`. Valgrind support is in its initial development -- please report any issues. @@ -523,7 +584,7 @@ To build with the address sanitizer, use the `-DMI_TRACK_ASAN=ON` cmake option: > cmake ../.. -DMI_TRACK_ASAN=ON ``` -This can also be combined with secure mode or debug mode. +This can also be combined with secure mode or debug mode. You can then run your programs as:' ``` @@ -531,7 +592,7 @@ You can then run your programs as:' ``` When you link a program with an address sanitizer build of mimalloc, you should -generally compile that program too with the address sanitizer enabled. +generally compile that program too with the address sanitizer enabled. For example, assuming you build mimalloc in `out/debug`: ``` @@ -540,23 +601,23 @@ clang -g -o test-wrong -Iinclude test/test-wrong.c out/debug/libmimalloc-asan-de Since the address sanitizer redirects the standard allocation functions, on some platforms (macOSX for example) it is required to compile mimalloc with `-DMI_OVERRIDE=OFF`. -Adress sanitizer support is in its initial development -- please report any issues. +Address sanitizer support is in its initial development -- please report any issues. [asan]: https://github.com/google/sanitizers/wiki/AddressSanitizer ## ETW Event tracing for Windows ([ETW]) provides a high performance way to capture all allocations though -mimalloc and analyze them later. To build with ETW support, use the `-DMI_TRACK_ETW=ON` cmake option. +mimalloc and analyze them later. To build with ETW support, use the `-DMI_TRACK_ETW=ON` cmake option. -You can then capture an allocation trace using the Windows performance recorder (WPR), using the +You can then capture an allocation trace using the Windows performance recorder (WPR), using the `src/prim/windows/etw-mimalloc.wprp` profile. In an admin prompt, you can use: ``` > wpr -start src\prim\windows\etw-mimalloc.wprp -filemode > > wpr -stop .etl -``` -and then open `.etl` in the Windows Performance Analyzer (WPA), or +``` +and then open `.etl` in the Windows Performance Analyzer (WPA), or use a tool like [TraceControl] that is specialized for analyzing mimalloc traces. [ETW]: https://learn.microsoft.com/en-us/windows-hardware/test/wpt/event-tracing-for-windows diff --git a/third-party/mimalloc/src/alloc-aligned.c b/third-party/mimalloc/src/alloc-aligned.c index ba629ef30a..8bf0a38dbb 100644 --- a/third-party/mimalloc/src/alloc-aligned.c +++ b/third-party/mimalloc/src/alloc-aligned.c @@ -24,6 +24,33 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0); } +#if MI_GUARDED +static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, size_t size, size_t alignment, bool zero) mi_attr_noexcept { + // use over allocation for guarded blocksl + mi_assert_internal(alignment > 0 && alignment < MI_BLOCK_ALIGNMENT_MAX); + const size_t oversize = size + alignment - 1; + void* base = _mi_heap_malloc_guarded(heap, oversize, zero); + void* p = mi_align_up_ptr(base, alignment); + mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size); + mi_assert_internal(mi_usable_size(p) >= size); + mi_assert_internal(_mi_is_aligned(p, alignment)); + return p; +} + +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { + const size_t rate = heap->guarded_sample_rate; + // only write if `rate!=0` so we don't write to the constant `_mi_heap_empty` + if (rate != 0) { heap->guarded_sample_rate = 0; } + void* p = _mi_heap_malloc_zero(heap, size, zero); + if (rate != 0) { heap->guarded_sample_rate = rate; } + return p; +} +#else +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { + return _mi_heap_malloc_zero(heap, size, zero); +} +#endif + // Fallback aligned allocation that over-allocates -- split out for better codegen static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept { @@ -38,22 +65,24 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t // first (and single) page such that the segment info is `MI_SEGMENT_SIZE` bytes before it (so it can be found by aligning the pointer down) if mi_unlikely(offset != 0) { // todo: cannot support offset alignment for very large alignments yet - #if MI_DEBUG > 0 +#if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); - #endif +#endif return NULL; } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); + // note: no guarded as alignment > 0 p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block // zero afterwards as only the area from the aligned_p may be committed! if (p == NULL) return NULL; } else { // otherwise over-allocate - oversize = size + alignment - 1; - p = _mi_heap_malloc_zero(heap, oversize, zero); + oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and aligment 64k, we would allocate a 64k block and pointing just beyond that. + p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero); if (p == NULL) return NULL; } + mi_page_t* page = _mi_ptr_page(p); // .. and align within the allocation const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` @@ -62,17 +91,27 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_assert_internal(adjust < alignment); void* aligned_p = (void*)((uintptr_t)p + adjust); if (aligned_p != p) { - mi_page_t* page = _mi_ptr_page(p); mi_page_set_has_aligned(page, true); + #if MI_GUARDED + // set tag to aligned so mi_usable_size works with guard pages + if (adjust >= sizeof(mi_block_t)) { + mi_block_t* const block = (mi_block_t*)p; + block->next = MI_BLOCK_TAG_ALIGNED; + } + #endif _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); } // todo: expand padding if overallocated ? - mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size); - mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_page(aligned_p), aligned_p)); + mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size); mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); + #if MI_DEBUG > 1 + mi_page_t* const apage = _mi_ptr_page(aligned_p); + void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == unalign_p); + #endif // now zero the block if needed if (alignment > MI_BLOCK_ALIGNMENT_MAX) { @@ -85,6 +124,9 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); + #if MI_GUARDED + mi_track_mem_defined(p, sizeof(mi_block_t)); + #endif } return aligned_p; } @@ -94,27 +136,27 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* { mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) - if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { + if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } - + // use regular allocation if it is guaranteed to fit the alignment constraints. // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = _mi_heap_malloc_zero(heap, size, zero); + void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); - const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { return p; } else { // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. mi_assert(false); - mi_free(p); + mi_free(p); } } @@ -122,6 +164,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* return mi_heap_malloc_zero_aligned_at_overalloc(heap,size,alignment,offset,zero); } + // Primitive aligned allocation static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept { @@ -132,11 +175,17 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t #endif return NULL; } - + + #if MI_GUARDED + if (offset==0 && alignment < MI_BLOCK_ALIGNMENT_MAX && mi_heap_malloc_use_guarded(heap,size)) { + return mi_heap_malloc_guarded_aligned(heap, size, alignment, zero); + } + #endif + // try first if there happens to be a small block available with just the right alignment if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - const size_t padsize = size + MI_PADDING_SIZE; + const size_t padsize = size + MI_PADDING_SIZE; mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize); if mi_likely(page->free != NULL) { const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; @@ -310,3 +359,5 @@ mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment); } + + diff --git a/third-party/mimalloc/src/alloc-override.c b/third-party/mimalloc/src/alloc-override.c index 12837cdd94..b5109ded0a 100644 --- a/third-party/mimalloc/src/alloc-override.c +++ b/third-party/mimalloc/src/alloc-override.c @@ -248,7 +248,7 @@ extern "C" { // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) - #if !defined(__ANDROID__) && !defined(__FreeBSD__) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) @@ -289,8 +289,8 @@ mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_r void __libc_free(void* p) MI_FORWARD0(mi_free, p) void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -#elif defined(__GLIBC__) && defined(__linux__) - // forward __libc interface (needed for glibc-based Linux distributions) +#elif defined(__linux__) + // forward __libc interface (needed for glibc-based and musl-based Linux distributions) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) diff --git a/third-party/mimalloc/src/alloc.c b/third-party/mimalloc/src/alloc.c index 86aaae757b..ffa7b8b70d 100644 --- a/third-party/mimalloc/src/alloc.c +++ b/third-party/mimalloc/src/alloc.c @@ -28,20 +28,25 @@ terms of the MIT license. A copy of the license can be found in the file // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept +extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size); + + // check the free list mi_block_t* const block = page->free; if mi_unlikely(block == NULL) { return _mi_malloc_generic(heap, size, zero, 0); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); + // pop from the free list page->free = mi_block_next(page, block); page->used++; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); + mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + #if MI_DEBUG>3 - if (page->free_is_zero) { + if (page->free_is_zero && size > sizeof(*block)) { mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); } #endif @@ -54,7 +59,10 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ // zero the block? note: we need to zero the full block size (issue #63) if mi_unlikely(zero) { mi_assert_internal(page->block_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic) + mi_assert_internal(!mi_page_is_huge(page)); + #if MI_PADDING mi_assert_internal(page->block_size >= MI_PADDING_SIZE); + #endif if (page->free_is_zero) { block->next = 0; mi_track_mem_defined(block, page->block_size - MI_PADDING_SIZE); @@ -91,7 +99,7 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); #endif mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess - padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); + padding->canary = mi_ptr_encode_canary(page,block,page->keys); padding->delta = (uint32_t)(delta); #if MI_PADDING_CHECK if (!mi_page_is_huge(page)) { @@ -113,19 +121,29 @@ extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t siz return _mi_page_malloc_zero(heap,page,size,true); } +#if MI_GUARDED +mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; +#endif + static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { mi_assert(heap != NULL); + mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG const uintptr_t tid = _mi_thread_id(); mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local #endif - mi_assert(size <= MI_SMALL_SIZE_MAX); - #if (MI_PADDING) + #if (MI_PADDING || MI_GUARDED) if (size == 0) { size = sizeof(void*); } #endif + #if MI_GUARDED + if (mi_heap_malloc_use_guarded(heap,size)) { + return _mi_heap_malloc_guarded(heap, size, zero); + } + #endif + // get page in constant time, and allocate from it mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); - void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero); + void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero); mi_track_malloc(p,size,zero); #if MI_STAT>1 @@ -153,15 +171,23 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t si // The main allocation function extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept { + // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); return mi_heap_malloc_small_zero(heap, size, zero); } + #if MI_GUARDED + else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) { + return _mi_heap_malloc_guarded(heap, size, zero); + } + #endif else { + // regular allocation mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p,size,zero); + #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } @@ -362,7 +388,7 @@ mi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_ #ifndef PATH_MAX #define PATH_MAX MAX_PATH #endif -#include + mi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names char buf[PATH_MAX]; @@ -530,7 +556,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si } mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { - return mi_heap_alloc_new_n(mi_prim_get_default_heap(), size, count); + return mi_heap_alloc_new_n(mi_prim_get_default_heap(), count, size); } @@ -577,6 +603,82 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } } +#if MI_GUARDED +// We always allocate a guarded allocation at an offset (`mi_page_has_aligned` will be true). +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` + +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) { + // TODO: we can still make padding work by moving it out of the guard page area + mi_page_t* const page = _mi_ptr_page(block); + mi_page_set_has_aligned(page, true); + block->next = MI_BLOCK_TAG_GUARDED; + + // set guard page at the end of the block + mi_segment_t* const segment = _mi_page_segment(page); + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); + if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { + // should never happen + mi_free(block); + return NULL; + } + uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); + if (segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) { + _mi_os_protect(guard_page, os_page_size); + } + else { + _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); + } + + // align pointer just in front of the guard page + size_t offset = block_size - os_page_size - obj_size; + mi_assert_internal(offset > sizeof(mi_block_t)); + if (offset > MI_BLOCK_ALIGNMENT_MAX) { + // give up to place it right in front of the guard page if the offset is too large for unalignment + offset = MI_BLOCK_ALIGNMENT_MAX; + } + void* p = (uint8_t*)block + offset; + mi_track_align(block, p, offset, obj_size); + mi_track_mem_defined(block, sizeof(mi_block_t)); + return p; +} + +mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept +{ + #if defined(MI_PADDING_SIZE) + mi_assert(MI_PADDING_SIZE==0); + #endif + // allocate multiple of page size ending in a guard page + // ensure minimal alignment requirement? + const size_t os_page_size = _mi_os_page_size(); + const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */); + if (block==NULL) return NULL; + void* const p = mi_block_ptr_set_guarded(block, obj_size); + + // stats + mi_track_malloc(p, size, zero); + if (p != NULL) { + if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } + #if MI_STAT>1 + mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + #endif + _mi_stat_counter_increase(&heap->tld->stats.guarded_alloc_count, 1); + } + #if MI_DEBUG>3 + if (p != NULL && zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; +} +#endif + // ------------------------------------------------------ // ensure explicit external inline definitions are emitted! // ------------------------------------------------------ @@ -584,6 +686,7 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { #ifdef __cplusplus void* _mi_externs[] = { (void*)&_mi_page_malloc, + (void*)&_mi_page_malloc_zero, (void*)&_mi_heap_malloc_zero, (void*)&_mi_heap_malloc_zero_ex, (void*)&mi_malloc, diff --git a/third-party/mimalloc/src/arena-abandon.c b/third-party/mimalloc/src/arena-abandon.c new file mode 100644 index 0000000000..460c80fc22 --- /dev/null +++ b/third-party/mimalloc/src/arena-abandon.c @@ -0,0 +1,346 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2024, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#if !defined(MI_IN_ARENA_C) +#error "this file should be included from 'arena.c' (so mi_arena_t is visible)" +// add includes help an IDE +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "bitmap.h" +#endif + +// Minimal exports for arena-abandoned. +size_t mi_arena_id_index(mi_arena_id_t id); +mi_arena_t* mi_arena_from_index(size_t idx); +size_t mi_arena_get_count(void); +void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex); +bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index); + +/* ----------------------------------------------------------- + Abandoned blocks/segments: + + _mi_arena_segment_clear_abandoned + _mi_arena_segment_mark_abandoned + + This is used to atomically abandon/reclaim segments + (and crosses the arena API but it is convenient to have here). + + Abandoned segments still have live blocks; they get reclaimed + when a thread frees a block in it, or when a thread needs a fresh + segment. + + Abandoned segments are atomically marked in the `block_abandoned` + bitmap of arenas. Any segments allocated outside arenas are put + in the sub-process `abandoned_os_list`. This list is accessed + using locks but this should be uncommon and generally uncontended. + Reclaim and visiting either scan through the `block_abandoned` + bitmaps of the arena's, or visit the `abandoned_os_list` + + A potentially nicer design is to use arena's for everything + and perhaps have virtual arena's to map OS allocated memory + but this would lack the "density" of our current arena's. TBC. +----------------------------------------------------------- */ + + +// reclaim a specific OS abandoned segment; `true` on success. +// sets the thread_id. +static bool mi_arena_segment_os_clear_abandoned(mi_segment_t* segment, bool take_lock) { + mi_assert(segment->memid.memkind != MI_MEM_ARENA); + // not in an arena, remove from list of abandoned os segments + mi_subproc_t* const subproc = segment->subproc; + if (take_lock && !mi_lock_try_acquire(&subproc->abandoned_os_lock)) { + return false; // failed to acquire the lock, we just give up + } + // remove atomically from the abandoned os list (if possible!) + bool reclaimed = false; + mi_segment_t* const next = segment->abandoned_os_next; + mi_segment_t* const prev = segment->abandoned_os_prev; + if (next != NULL || prev != NULL || subproc->abandoned_os_list == segment) { + #if MI_DEBUG>3 + // find ourselves in the abandoned list (and check the count) + bool found = false; + size_t count = 0; + for (mi_segment_t* current = subproc->abandoned_os_list; current != NULL; current = current->abandoned_os_next) { + if (current == segment) { found = true; } + count++; + } + mi_assert_internal(found); + mi_assert_internal(count == mi_atomic_load_relaxed(&subproc->abandoned_os_list_count)); + #endif + // remove (atomically) from the list and reclaim + if (prev != NULL) { prev->abandoned_os_next = next; } + else { subproc->abandoned_os_list = next; } + if (next != NULL) { next->abandoned_os_prev = prev; } + else { subproc->abandoned_os_list_tail = prev; } + segment->abandoned_os_next = NULL; + segment->abandoned_os_prev = NULL; + mi_atomic_decrement_relaxed(&subproc->abandoned_count); + mi_atomic_decrement_relaxed(&subproc->abandoned_os_list_count); + if (take_lock) { // don't reset the thread_id when iterating + mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); + } + reclaimed = true; + } + if (take_lock) { mi_lock_release(&segment->subproc->abandoned_os_lock); } + return reclaimed; +} + +// reclaim a specific abandoned segment; `true` on success. +// sets the thread_id. +bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment) { + if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) { + return mi_arena_segment_os_clear_abandoned(segment, true /* take lock */); + } + // arena segment: use the blocks_abandoned bitmap. + size_t arena_idx; + size_t bitmap_idx; + mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); + mi_arena_t* arena = mi_arena_from_index(arena_idx); + mi_assert_internal(arena != NULL); + // reclaim atomically + bool was_marked = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); + if (was_marked) { + mi_assert_internal(mi_atomic_load_acquire(&segment->thread_id) == 0); + mi_atomic_decrement_relaxed(&segment->subproc->abandoned_count); + mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); + } + // mi_assert_internal(was_marked); + mi_assert_internal(!was_marked || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); + //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); + return was_marked; +} + + +// mark a specific OS segment as abandoned +static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) { + mi_assert(segment->memid.memkind != MI_MEM_ARENA); + // not in an arena; we use a list of abandoned segments + mi_subproc_t* const subproc = segment->subproc; + mi_lock(&subproc->abandoned_os_lock) { + // push on the tail of the list (important for the visitor) + mi_segment_t* prev = subproc->abandoned_os_list_tail; + mi_assert_internal(prev == NULL || prev->abandoned_os_next == NULL); + mi_assert_internal(segment->abandoned_os_prev == NULL); + mi_assert_internal(segment->abandoned_os_next == NULL); + if (prev != NULL) { prev->abandoned_os_next = segment; } + else { subproc->abandoned_os_list = segment; } + subproc->abandoned_os_list_tail = segment; + segment->abandoned_os_prev = prev; + segment->abandoned_os_next = NULL; + mi_atomic_increment_relaxed(&subproc->abandoned_os_list_count); + mi_atomic_increment_relaxed(&subproc->abandoned_count); + // and release the lock + } + return; +} + +// mark a specific segment as abandoned +// clears the thread_id. +void _mi_arena_segment_mark_abandoned(mi_segment_t* segment) +{ + mi_assert_internal(segment->used == segment->abandoned); + mi_atomic_store_release(&segment->thread_id, (uintptr_t)0); // mark as abandoned for multi-thread free's + if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) { + mi_arena_segment_os_mark_abandoned(segment); + return; + } + // segment is in an arena, mark it in the arena `blocks_abandoned` bitmap + size_t arena_idx; + size_t bitmap_idx; + mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); + mi_arena_t* arena = mi_arena_from_index(arena_idx); + mi_assert_internal(arena != NULL); + // set abandonment atomically + mi_subproc_t* const subproc = segment->subproc; // don't access the segment after setting it abandoned + const bool was_unmarked = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); + if (was_unmarked) { mi_atomic_increment_relaxed(&subproc->abandoned_count); } + mi_assert_internal(was_unmarked); + mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); +} + + +/* ----------------------------------------------------------- + Iterate through the abandoned blocks/segments using a cursor. + This is used for reclaiming and abandoned block visiting. +----------------------------------------------------------- */ + +// start a cursor at a randomized arena +void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool visit_all, mi_arena_field_cursor_t* current) { + mi_assert_internal(heap == NULL || heap->tld->segments.subproc == subproc); + current->bitmap_idx = 0; + current->subproc = subproc; + current->visit_all = visit_all; + current->hold_visit_lock = false; + const size_t abandoned_count = mi_atomic_load_relaxed(&subproc->abandoned_count); + const size_t abandoned_list_count = mi_atomic_load_relaxed(&subproc->abandoned_os_list_count); + const size_t max_arena = mi_arena_get_count(); + if (heap != NULL && heap->arena_id != _mi_arena_id_none()) { + // for a heap that is bound to one arena, only visit that arena + current->start = mi_arena_id_index(heap->arena_id); + current->end = current->start + 1; + current->os_list_count = 0; + } + else { + // otherwise visit all starting at a random location + if (abandoned_count > abandoned_list_count && max_arena > 0) { + current->start = (heap == NULL || max_arena == 0 ? 0 : (mi_arena_id_t)(_mi_heap_random_next(heap) % max_arena)); + current->end = current->start + max_arena; + } + else { + current->start = 0; + current->end = 0; + } + current->os_list_count = abandoned_list_count; // max entries to visit in the os abandoned list + } + mi_assert_internal(current->start <= max_arena); +} + +void _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current) { + if (current->hold_visit_lock) { + mi_lock_release(¤t->subproc->abandoned_os_visit_lock); + current->hold_visit_lock = false; + } +} + +static mi_segment_t* mi_arena_segment_clear_abandoned_at(mi_arena_t* arena, mi_subproc_t* subproc, mi_bitmap_index_t bitmap_idx) { + // try to reclaim an abandoned segment in the arena atomically + if (!_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) return NULL; + mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); + mi_segment_t* segment = (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx); + mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); + // check that the segment belongs to our sub-process + // note: this is the reason we need the `abandoned_visit` lock in the case abandoned visiting is enabled. + // without the lock an abandoned visit may otherwise fail to visit all abandoned segments in the sub-process. + // for regular reclaim it is fine to miss one sometimes so without abandoned visiting we don't need the `abandoned_visit` lock. + if (segment->subproc != subproc) { + // it is from another sub-process, re-mark it and continue searching + const bool was_zero = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); + mi_assert_internal(was_zero); MI_UNUSED(was_zero); + return NULL; + } + else { + // success, we unabandoned a segment in our sub-process + mi_atomic_decrement_relaxed(&subproc->abandoned_count); + return segment; + } +} + +static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_cursor_t* previous) { + const size_t max_arena = mi_arena_get_count(); + size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx); + size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx); + // visit arena's (from the previous cursor) + for (; previous->start < previous->end; previous->start++, field_idx = 0, bit_idx = 0) { + // index wraps around + size_t arena_idx = (previous->start >= max_arena ? previous->start % max_arena : previous->start); + mi_arena_t* arena = mi_arena_from_index(arena_idx); + if (arena != NULL) { + bool has_lock = false; + // visit the abandoned fields (starting at previous_idx) + for (; field_idx < arena->field_count; field_idx++, bit_idx = 0) { + size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]); + if mi_unlikely(field != 0) { // skip zero fields quickly + // we only take the arena lock if there are actually abandoned segments present + if (!has_lock && mi_option_is_enabled(mi_option_visit_abandoned)) { + has_lock = (previous->visit_all ? (mi_lock_acquire(&arena->abandoned_visit_lock),true) : mi_lock_try_acquire(&arena->abandoned_visit_lock)); + if (!has_lock) { + if (previous->visit_all) { + _mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the visitor lock"); + } + // skip to next arena + break; + } + } + mi_assert_internal(has_lock || !mi_option_is_enabled(mi_option_visit_abandoned)); + // visit each set bit in the field (todo: maybe use `ctz` here?) + for (; bit_idx < MI_BITMAP_FIELD_BITS; bit_idx++) { + // pre-check if the bit is set + size_t mask = ((size_t)1 << bit_idx); + if mi_unlikely((field & mask) == mask) { + mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); + mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, bitmap_idx); + if (segment != NULL) { + //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); + if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); } + previous->bitmap_idx = mi_bitmap_index_create_ex(field_idx, bit_idx + 1); // start at next one for the next iteration + return segment; + } + } + } + } + } + if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); } + } + } + return NULL; +} + +static mi_segment_t* mi_arena_segment_clear_abandoned_next_list(mi_arena_field_cursor_t* previous) { + // go through the abandoned_os_list + // we only allow one thread per sub-process to do to visit guarded by the `abandoned_os_visit_lock`. + // The lock is released when the cursor is released. + if (!previous->hold_visit_lock) { + previous->hold_visit_lock = (previous->visit_all ? (mi_lock_acquire(&previous->subproc->abandoned_os_visit_lock),true) + : mi_lock_try_acquire(&previous->subproc->abandoned_os_visit_lock)); + if (!previous->hold_visit_lock) { + if (previous->visit_all) { + _mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the OS visitor lock"); + } + return NULL; // we cannot get the lock, give up + } + } + // One list entry at a time + while (previous->os_list_count > 0) { + previous->os_list_count--; + mi_lock_acquire(&previous->subproc->abandoned_os_lock); // this could contend with concurrent OS block abandonment and reclaim from `free` + mi_segment_t* segment = previous->subproc->abandoned_os_list; + // pop from head of the list, a subsequent mark will push at the end (and thus we iterate through os_list_count entries) + if (segment == NULL || mi_arena_segment_os_clear_abandoned(segment, false /* we already have the lock */)) { + mi_lock_release(&previous->subproc->abandoned_os_lock); + return segment; + } + // already abandoned, try again + mi_lock_release(&previous->subproc->abandoned_os_lock); + } + // done + mi_assert_internal(previous->os_list_count == 0); + return NULL; +} + + +// reclaim abandoned segments +// this does not set the thread id (so it appears as still abandoned) +mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous) { + if (previous->start < previous->end) { + // walk the arena + mi_segment_t* segment = mi_arena_segment_clear_abandoned_next_field(previous); + if (segment != NULL) { return segment; } + } + // no entries in the arena's anymore, walk the abandoned OS list + mi_assert_internal(previous->start == previous->end); + return mi_arena_segment_clear_abandoned_next_list(previous); +} + + +bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + // (unfortunately) the visit_abandoned option must be enabled from the start. + // This is to avoid taking locks if abandoned list visiting is not required (as for most programs) + if (!mi_option_is_enabled(mi_option_visit_abandoned)) { + _mi_error_message(EFAULT, "internal error: can only visit abandoned blocks when MIMALLOC_VISIT_ABANDONED=ON"); + return false; + } + mi_arena_field_cursor_t current; + _mi_arena_field_cursor_init(NULL, _mi_subproc_from_id(subproc_id), true /* visit all (blocking) */, ¤t); + mi_segment_t* segment; + bool ok = true; + while (ok && (segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) { + ok = _mi_segment_visit_blocks(segment, heap_tag, visit_blocks, visitor, arg); + _mi_arena_segment_mark_abandoned(segment); + } + _mi_arena_field_cursor_done(¤t); + return ok; +} diff --git a/third-party/mimalloc/src/arena.c b/third-party/mimalloc/src/arena.c index 648ee844fe..a809b85f46 100644 --- a/third-party/mimalloc/src/arena.c +++ b/third-party/mimalloc/src/arena.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2023, Microsoft Research, Daan Leijen +Copyright (c) 2019-2024, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -11,69 +11,68 @@ large blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 4MiB). In contrast to the rest of mimalloc, the arenas are shared between threads and need to be accessed using atomic operations. -Arenas are used to for huge OS page (1GiB) reservations or for reserving +Arenas are also used to for huge OS page (1GiB) reservations or for reserving OS memory upfront which can be improve performance or is sometimes needed on embedded devices. We can also employ this with WASI or `sbrk` systems to reserve large arenas upfront and be able to reuse the memory more effectively. The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. -----------------------------------------------------------------------------*/ + #include "mimalloc.h" #include "mimalloc/internal.h" #include "mimalloc/atomic.h" +#include "bitmap.h" -#include // memset -#include // ENOMEM - -#include "bitmap.h" // atomic bitmap /* ----------------------------------------------------------- Arena allocation ----------------------------------------------------------- */ -// Block info: bit 0 contains the `in_use` bit, the upper bits the -// size in count of arena blocks. -typedef uintptr_t mi_block_info_t; -#define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 64MiB (must be at least MI_SEGMENT_ALIGN) -#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 32MiB -#define MI_MAX_ARENAS (112) // not more than 126 (since we use 7 bits in the memid and an arena index + 1) - // A memory arena descriptor typedef struct mi_arena_s { - mi_arena_id_t id; // arena id; 0 for non-specific - mi_memid_t memid; // memid of the memory area - _Atomic(uint8_t*) start; // the start of the memory area - size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) - size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) - size_t meta_size; // size of the arena structure itself (including its bitmaps) - mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation) - int numa_node; // associated NUMA node - bool exclusive; // only allow allocations if specifically for this arena - bool is_large; // memory area consists of large- or huge OS pages (always committed) - _Atomic(size_t) search_idx; // optimization to start the search for free blocks - _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. - mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? - mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) - mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) - mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here) - mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) + mi_arena_id_t id; // arena id; 0 for non-specific + mi_memid_t memid; // memid of the memory area + _Atomic(uint8_t*) start; // the start of the memory area + size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) + size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) + size_t meta_size; // size of the arena structure itself (including its bitmaps) + mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation) + int numa_node; // associated NUMA node + bool exclusive; // only allow allocations if specifically for this arena + bool is_large; // memory area consists of large- or huge OS pages (always committed) + mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited + _Atomic(size_t) search_idx; // optimization to start the search for free blocks + _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be purged from `blocks_purge`. + + mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) + mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) + mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here) + mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) // do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields. } mi_arena_t; +#define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 64MiB (must be at least MI_SEGMENT_ALIGN) +#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 32MiB +#define MI_MAX_ARENAS (132) // Limited as the reservation exponentially increases (and takes up .bss) + // The available arenas static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS]; static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 +static mi_decl_cache_align _Atomic(int64_t) mi_arenas_purge_expire; // set if there exist purgeable arenas - -//static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept; +#define MI_IN_ARENA_C +#include "arena-abandon.c" +#undef MI_IN_ARENA_C /* ----------------------------------------------------------- Arena id's id = arena_index + 1 ----------------------------------------------------------- */ -static size_t mi_arena_id_index(mi_arena_id_t id) { +size_t mi_arena_id_index(mi_arena_id_t id) { return (size_t)(id <= 0 ? MI_MAX_ARENAS : id - 1); } @@ -104,6 +103,16 @@ bool _mi_arena_memid_is_os_allocated(mi_memid_t memid) { return (memid.memkind == MI_MEM_OS); } +size_t mi_arena_get_count(void) { + return mi_atomic_load_relaxed(&mi_arena_count); +} + +mi_arena_t* mi_arena_from_index(size_t idx) { + mi_assert_internal(idx < mi_arena_get_count()); + return mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[idx]); +} + + /* ----------------------------------------------------------- Arena allocations get a (currently) 16-bit memory id where the lower 8 bits are the arena id, and the upper bits the block index. @@ -129,7 +138,7 @@ static mi_memid_t mi_memid_create_arena(mi_arena_id_t id, bool is_exclusive, mi_ return memid; } -static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { +bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) { mi_assert_internal(memid.memkind == MI_MEM_ARENA); *arena_index = mi_arena_id_index(memid.mem.arena.id); *bitmap_index = memid.mem.arena.block_index; @@ -140,10 +149,10 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit /* ----------------------------------------------------------- Special static area for mimalloc internal structures - to avoid OS calls (for example, for the arena metadata) + to avoid OS calls (for example, for the arena metadata (~= 256b)) ----------------------------------------------------------- */ -#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit +#define MI_ARENA_STATIC_MAX ((MI_INTPTR_SIZE/2)*MI_KiB) // 4 KiB on 64-bit static mi_decl_cache_align uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; // must be cache aligned, see issue #895 static mi_decl_cache_align _Atomic(size_t) mi_arena_static_top; @@ -175,7 +184,7 @@ static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* m return p; } -static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* stats) { +void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) { *memid = _mi_memid_none(); // try static @@ -183,7 +192,7 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st if (p != NULL) return p; // or fall back to the OS - p = _mi_os_alloc(size, memid, stats); + p = _mi_os_alloc(size, memid); if (p == NULL) return NULL; // zero the OS memory if needed @@ -194,16 +203,16 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st return p; } -static void mi_arena_meta_free(void* p, mi_memid_t memid, size_t size, mi_stats_t* stats) { +void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size) { if (mi_memkind_is_os(memid.memkind)) { - _mi_os_free(p, size, memid, stats); + _mi_os_free(p, size, memid); } else { mi_assert(memid.memkind == MI_MEM_STATIC); } } -static void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) { +void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) { return (arena->start + mi_arena_block_size(mi_bitmap_index_bit(bindex))); } @@ -213,10 +222,10 @@ static void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) { ----------------------------------------------------------- */ // claim the `blocks_inuse` bits -static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats) +static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx) { size_t idx = 0; // mi_atomic_load_relaxed(&arena->search_idx); // start from last search; ok to be relaxed as the exact start does not matter - if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx, stats)) { + if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx)) { mi_atomic_store_relaxed(&arena->search_idx, mi_bitmap_index_field(*bitmap_idx)); // start search from found location next time around return true; }; @@ -229,13 +238,13 @@ static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index ----------------------------------------------------------- */ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t arena_index, size_t needed_bcount, - bool commit, mi_memid_t* memid, mi_os_tld_t* tld) + bool commit, mi_memid_t* memid) { MI_UNUSED(arena_index); mi_assert_internal(mi_arena_id_index(arena->id) == arena_index); mi_bitmap_index_t bitmap_index; - if (!mi_arena_try_claim(arena, needed_bcount, &bitmap_index, tld->stats)) return NULL; + if (!mi_arena_try_claim(arena, needed_bcount, &bitmap_index)) return NULL; // claimed it! void* p = mi_arena_block_start(arena, bitmap_index); @@ -265,7 +274,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar _mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted); if (any_uncommitted) { bool commit_zero = false; - if (!_mi_os_commit(p, mi_arena_block_size(needed_bcount), &commit_zero, tld->stats)) { + if (!_mi_os_commit(p, mi_arena_block_size(needed_bcount), &commit_zero)) { memid->initially_committed = false; } else { @@ -281,19 +290,19 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar return p; } -// allocate in a speficic arena +// allocate in a specific arena static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_node, int numa_node, size_t size, size_t alignment, - bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) + bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid ) { MI_UNUSED_RELEASE(alignment); - mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); + mi_assert(alignment <= MI_SEGMENT_ALIGN); const size_t bcount = mi_block_count_of_size(size); const size_t arena_index = mi_arena_id_index(arena_id); mi_assert_internal(arena_index < mi_atomic_load_relaxed(&mi_arena_count)); mi_assert_internal(size <= mi_arena_block_size(bcount)); // Check arena suitability - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_index]); + mi_arena_t* arena = mi_arena_from_index(arena_index); if (arena == NULL) return NULL; if (!allow_large && arena->is_large) return NULL; if (!mi_arena_id_is_suitable(arena->id, arena->exclusive, req_arena_id)) return NULL; @@ -304,7 +313,7 @@ static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_no } // try to allocate - void* p = mi_arena_try_alloc_at(arena, arena_index, bcount, commit, memid, tld); + void* p = mi_arena_try_alloc_at(arena, arena_index, bcount, commit, memid); mi_assert_internal(p == NULL || _mi_is_aligned(p, alignment)); return p; } @@ -313,7 +322,7 @@ static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_no // allocate from an arena with fallback to the OS static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, size_t alignment, bool commit, bool allow_large, - mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) + mi_arena_id_t req_arena_id, mi_memid_t* memid ) { MI_UNUSED(alignment); mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); @@ -323,21 +332,21 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz if (req_arena_id != _mi_arena_id_none()) { // try a specific arena if requested if (mi_arena_id_index(req_arena_id) < max_arena) { - void* p = mi_arena_try_alloc_at_id(req_arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + void* p = mi_arena_try_alloc_at_id(req_arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; } } else { // try numa affine allocation for (size_t i = 0; i < max_arena; i++) { - void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; } // try from another numa node instead.. if (numa_node >= 0) { // if numa_node was < 0 (no specific affinity requested), all arena's have been tried already for (size_t i = 0; i < max_arena; i++) { - void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), false /* only proceed if not numa local */, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), false /* only proceed if not numa local */, numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; } } @@ -346,11 +355,10 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz } // try to reserve a fresh arena space -static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t req_arena_id, mi_arena_id_t *arena_id) +static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t *arena_id) { if (_mi_preloading()) return false; // use OS only while pre loading - if (req_arena_id != _mi_arena_id_none()) return false; - + const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count); if (arena_count > (MI_MAX_ARENAS - 4)) return false; @@ -361,8 +369,14 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example) } arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_BLOCK_SIZE); + arena_reserve = _mi_align_up(arena_reserve, MI_SEGMENT_SIZE); if (arena_count >= 8 && arena_count <= 128) { - arena_reserve = ((size_t)1<<(arena_count/8)) * arena_reserve; // scale up the arena sizes exponentially + // scale up the arena sizes exponentially every 8 entries (128 entries get to 589TiB) + const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16 ); + size_t reserve = 0; + if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) { + arena_reserve = reserve; + } } if (arena_reserve < req_size) return false; // should be able to at least handle the current allocation size @@ -376,27 +390,28 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, - mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld) + mi_arena_id_t req_arena_id, mi_memid_t* memid) { - mi_assert_internal(memid != NULL && tld != NULL); + mi_assert_internal(memid != NULL); mi_assert_internal(size > 0); *memid = _mi_memid_none(); - const int numa_node = _mi_os_numa_node(tld); // current numa node + const int numa_node = _mi_os_numa_node(); // current numa node // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data) - if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) || req_arena_id != _mi_arena_id_none()) { // is arena allocation allowed? - if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) { - void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + if (!mi_option_is_enabled(mi_option_disallow_arena_alloc)) { // is arena allocation allowed? + if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) + { + void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; // otherwise, try to first eagerly reserve a new arena if (req_arena_id == _mi_arena_id_none()) { mi_arena_id_t arena_id = 0; - if (mi_arena_reserve(size, allow_large, req_arena_id, &arena_id)) { + if (mi_arena_reserve(size, allow_large, &arena_id)) { // and try allocate in there mi_assert_internal(req_arena_id == _mi_arena_id_none()); - p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld); + p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; } } @@ -411,16 +426,16 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset // finally, fall back to the OS if (align_offset > 0) { - return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid, tld->stats); + return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid); } else { - return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid, tld->stats); + return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid); } } -void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld) +void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid) { - return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, 0, commit, allow_large, req_arena_id, memid, tld); + return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, 0, commit, allow_large, req_arena_id, memid); } @@ -446,7 +461,7 @@ static long mi_arena_purge_delay(void) { // reset or decommit in an arena and update the committed/decommit bitmaps // assumes we own the area (i.e. blocks_in_use is claimed by us) -static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, mi_stats_t* stats) { +static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) { mi_assert_internal(arena->blocks_committed != NULL); mi_assert_internal(arena->blocks_purge != NULL); mi_assert_internal(!arena->memid.is_pinned); @@ -455,7 +470,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, bool needs_recommit; if (_mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx)) { // all blocks are committed, we can purge freely - needs_recommit = _mi_os_purge(p, size, stats); + needs_recommit = _mi_os_purge(p, size); } else { // some blocks are not committed -- this can happen when a partially committed block is freed @@ -463,8 +478,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, // we need to ensure we do not try to reset (as that may be invalid for uncommitted memory), // and also undo the decommit stats (as it was already adjusted) mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits)); - needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, stats); - if (needs_recommit) { _mi_stat_increase(&_mi_stats_main.committed, size); } + needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, 0); } // clear the purged blocks @@ -477,23 +491,26 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, // Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls. // Note: assumes we (still) own the area as we may purge immediately -static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, mi_stats_t* stats) { +static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) { mi_assert_internal(arena->blocks_purge != NULL); const long delay = mi_arena_purge_delay(); if (delay < 0) return; // is purging allowed at all? if (_mi_preloading() || delay == 0) { // decommit directly - mi_arena_purge(arena, bitmap_idx, blocks, stats); + mi_arena_purge(arena, bitmap_idx, blocks); } else { - // schedule decommit - mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (expire != 0) { - mi_atomic_addi64_acq_rel(&arena->purge_expire, (mi_msecs_t)(delay/10)); // add smallish extra delay + // schedule purge + const mi_msecs_t expire = _mi_clock_now() + delay; + mi_msecs_t expire0 = 0; + if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) { + // expiration was not yet set + // maybe set the global arenas expire as well (if it wasn't set already) + mi_atomic_casi64_strong_acq_rel(&mi_arenas_purge_expire, &expire0, expire); } else { - mi_atomic_storei64_release(&arena->purge_expire, _mi_clock_now() + delay); + // already an expiration was set } _mi_bitmap_claim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx, NULL); } @@ -502,12 +519,12 @@ static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t // purge a range of blocks // return true if the full range was purged. // assumes we own the area (i.e. blocks_in_use is claimed by us) -static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitlen, size_t purge, mi_stats_t* stats) { +static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitlen, size_t purge) { const size_t endidx = startidx + bitlen; size_t bitidx = startidx; bool all_purged = false; while (bitidx < endidx) { - // count consequetive ones in the purge mask + // count consecutive ones in the purge mask size_t count = 0; while (bitidx + count < endidx && (purge & ((size_t)1 << (bitidx + count))) != 0) { count++; @@ -515,7 +532,7 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, if (count > 0) { // found range to be purged const mi_bitmap_index_t range_idx = mi_bitmap_index_create(idx, bitidx); - mi_arena_purge(arena, range_idx, count, stats); + mi_arena_purge(arena, range_idx, count); if (count == bitlen) { all_purged = true; } @@ -526,16 +543,18 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, } // returns true if anything was purged -static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi_stats_t* stats) +static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) { - if (arena->memid.is_pinned || arena->blocks_purge == NULL) return false; + // check pre-conditions + if (arena->memid.is_pinned) return false; + + // expired yet? mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (expire == 0) return false; - if (!force && expire > now) return false; + if (!force && (expire == 0 || expire > now)) return false; // reset expire (if not already set concurrently) mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0); - + // potential purges scheduled, walk through the bitmap bool any_purged = false; bool full_purge = true; @@ -544,11 +563,12 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi if (purge != 0) { size_t bitidx = 0; while (bitidx < MI_BITMAP_FIELD_BITS) { - // find consequetive range of ones in the purge mask + // find consecutive range of ones in the purge mask size_t bitlen = 0; while (bitidx + bitlen < MI_BITMAP_FIELD_BITS && (purge & ((size_t)1 << (bitidx + bitlen))) != 0) { bitlen++; } + // temporarily claim the purge range as "in-use" to be thread-safe with allocation // try to claim the longest range of corresponding in_use bits const mi_bitmap_index_t bitmap_index = mi_bitmap_index_create(i, bitidx); while( bitlen > 0 ) { @@ -561,7 +581,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi if (bitlen > 0) { // read purge again now that we have the in_use bits purge = mi_atomic_load_acquire(&arena->blocks_purge[i]); - if (!mi_arena_purge_range(arena, i, bitidx, bitlen, purge, stats)) { + if (!mi_arena_purge_range(arena, i, bitidx, bitlen, purge)) { full_purge = false; } any_purged = true; @@ -581,9 +601,15 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi return any_purged; } -static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats ) { +static void mi_arenas_try_purge( bool force, bool visit_all ) +{ if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled + // check if any arena needs purging? + const mi_msecs_t now = _mi_clock_now(); + mi_msecs_t arenas_expire = mi_atomic_load_acquire(&mi_arenas_purge_expire); + if (!force && (arenas_expire == 0 || arenas_expire < now)) return; + const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count); if (max_arena == 0) return; @@ -591,17 +617,26 @@ static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats ) static mi_atomic_guard_t purge_guard; mi_atomic_guard(&purge_guard) { - mi_msecs_t now = _mi_clock_now(); - size_t max_purge_count = (visit_all ? max_arena : 1); + // increase global expire: at most one purge per delay cycle + mi_atomic_store_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay()); + size_t max_purge_count = (visit_all ? max_arena : 2); + bool all_visited = true; for (size_t i = 0; i < max_arena; i++) { mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); if (arena != NULL) { - if (mi_arena_try_purge(arena, now, force, stats)) { - if (max_purge_count <= 1) break; + if (mi_arena_try_purge(arena, now, force)) { + if (max_purge_count <= 1) { + all_visited = false; + break; + } max_purge_count--; } } } + if (all_visited) { + // all arena's were visited and purged: reset global expire + mi_atomic_store_release(&mi_arenas_purge_expire, 0); + } } } @@ -610,20 +645,23 @@ static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats ) Arena free ----------------------------------------------------------- */ -void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memid, mi_stats_t* stats) { - mi_assert_internal(size > 0 && stats != NULL); +void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memid) { + mi_assert_internal(size > 0); mi_assert_internal(committed_size <= size); if (p==NULL) return; if (size==0) return; const bool all_committed = (committed_size == size); + // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) + mi_track_mem_undefined(p,size); + if (mi_memkind_is_os(memid.memkind)) { // was a direct OS allocation, pass through if (!all_committed && committed_size > 0) { // if partially committed, adjust the committed stats (as `_mi_os_free` will increase decommit by the full size) _mi_stat_decrease(&_mi_stats_main.committed, committed_size); } - _mi_os_free(p, size, memid, stats); + _mi_os_free(p, size, memid); } else if (memid.memkind == MI_MEM_ARENA) { // allocated in an arena @@ -646,9 +684,6 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi return; } - // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.) - mi_track_mem_undefined(p,size); - // potentially decommit if (arena->memid.is_pinned || arena->blocks_committed == NULL) { mi_assert_internal(all_committed); @@ -671,7 +706,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi // works (as we should never reset decommitted parts). } // (delay) purge the entire range - mi_arena_schedule_purge(arena, bitmap_idx, blocks, stats); + mi_arena_schedule_purge(arena, bitmap_idx, blocks); } // and make it available to others again @@ -687,7 +722,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi } // purge expired decommits - mi_arenas_try_purge(false, false, stats); + mi_arenas_try_purge(false, false); } // destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` @@ -698,14 +733,15 @@ static void mi_arenas_unsafe_destroy(void) { for (size_t i = 0; i < max_arena; i++) { mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); if (arena != NULL) { + mi_lock_done(&arena->abandoned_visit_lock); if (arena->start != NULL && mi_memkind_is_os(arena->memid.memkind)) { mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL); - _mi_os_free(arena->start, mi_arena_size(arena), arena->memid, &_mi_stats_main); + _mi_os_free(arena->start, mi_arena_size(arena), arena->memid); } else { new_max_arena = i; } - mi_arena_meta_free(arena, arena->meta_memid, arena->meta_size, &_mi_stats_main); + _mi_arena_meta_free(arena, arena->meta_memid, arena->meta_size); } } @@ -715,22 +751,22 @@ static void mi_arenas_unsafe_destroy(void) { } // Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired -void _mi_arenas_collect(bool force_purge, mi_stats_t* stats) { - mi_arenas_try_purge(force_purge, force_purge /* visit all? */, stats); +void _mi_arenas_collect(bool force_purge) { + mi_arenas_try_purge(force_purge, force_purge /* visit all? */); } // destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` // for dynamic libraries that are unloaded and need to release all their allocated memory. -void _mi_arena_unsafe_destroy_all(mi_stats_t* stats) { +void _mi_arena_unsafe_destroy_all(void) { mi_arenas_unsafe_destroy(); - _mi_arenas_collect(true /* force purge */, stats); // purge non-owned arenas + _mi_arenas_collect(true /* force purge */); // purge non-owned arenas } // Is a pointer inside any of our arenas? bool _mi_arena_contains(const void* p) { const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); for (size_t i = 0; i < max_arena; i++) { - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); if (arena != NULL && arena->start <= (const uint8_t*)p && arena->start + mi_arena_block_size(arena->block_count) > (const uint8_t*)p) { return true; } @@ -738,139 +774,6 @@ bool _mi_arena_contains(const void* p) { return false; } -/* ----------------------------------------------------------- - Abandoned blocks/segments. - This is used to atomically abandon/reclaim segments - (and crosses the arena API but it is convenient to have here). - Abandoned segments still have live blocks; they get reclaimed - when a thread frees a block in it, or when a thread needs a fresh - segment; these threads scan the abandoned segments through - the arena bitmaps. ------------------------------------------------------------ */ - -// Maintain a count of all abandoned segments -static mi_decl_cache_align _Atomic(size_t)abandoned_count; - -size_t _mi_arena_segment_abandoned_count(void) { - return mi_atomic_load_relaxed(&abandoned_count); -} - -// reclaim a specific abandoned segment; `true` on success. -// sets the thread_id. -bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment ) -{ - if (segment->memid.memkind != MI_MEM_ARENA) { - // not in an arena, consider it un-abandoned now. - // but we need to still claim it atomically -- we use the thread_id for that. - size_t expected = 0; - if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected, _mi_thread_id())) { - mi_atomic_decrement_relaxed(&abandoned_count); - return true; - } - else { - return false; - } - } - // arena segment: use the blocks_abandoned bitmap. - size_t arena_idx; - size_t bitmap_idx; - mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); - mi_assert_internal(arena_idx < MI_MAX_ARENAS); - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); - mi_assert_internal(arena != NULL); - bool was_marked = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); - if (was_marked) { - mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); - mi_atomic_decrement_relaxed(&abandoned_count); - mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); - } - // mi_assert_internal(was_marked); - mi_assert_internal(!was_marked || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); - //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); - return was_marked; -} - -// mark a specific segment as abandoned -// clears the thread_id. -void _mi_arena_segment_mark_abandoned(mi_segment_t* segment) -{ - mi_atomic_store_release(&segment->thread_id, 0); - mi_assert_internal(segment->used == segment->abandoned); - if (segment->memid.memkind != MI_MEM_ARENA) { - // not in an arena; count it as abandoned and return - mi_atomic_increment_relaxed(&abandoned_count); - return; - } - size_t arena_idx; - size_t bitmap_idx; - mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx); - mi_assert_internal(arena_idx < MI_MAX_ARENAS); - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); - mi_assert_internal(arena != NULL); - const bool was_unmarked = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); - if (was_unmarked) { mi_atomic_increment_relaxed(&abandoned_count); } - mi_assert_internal(was_unmarked); - mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); -} - -// start a cursor at a randomized arena -void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current) { - const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); - current->start = (max_arena == 0 ? 0 : (mi_arena_id_t)( _mi_heap_random_next(heap) % max_arena)); - current->count = 0; - current->bitmap_idx = 0; -} - -// reclaim abandoned segments -// this does not set the thread id (so it appears as still abandoned) -mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous ) -{ - const int max_arena = (int)mi_atomic_load_relaxed(&mi_arena_count); - if (max_arena <= 0 || mi_atomic_load_relaxed(&abandoned_count) == 0) return NULL; - - int count = previous->count; - size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx); - size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx) + 1; - // visit arena's (from previous) - for (; count < max_arena; count++, field_idx = 0, bit_idx = 0) { - mi_arena_id_t arena_idx = previous->start + count; - if (arena_idx >= max_arena) { arena_idx = arena_idx % max_arena; } // wrap around - mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); - if (arena != NULL) { - // visit the abandoned fields (starting at previous_idx) - for ( ; field_idx < arena->field_count; field_idx++, bit_idx = 0) { - size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]); - if mi_unlikely(field != 0) { // skip zero fields quickly - // visit each set bit in the field (todo: maybe use `ctz` here?) - for ( ; bit_idx < MI_BITMAP_FIELD_BITS; bit_idx++) { - // pre-check if the bit is set - size_t mask = ((size_t)1 << bit_idx); - if mi_unlikely((field & mask) == mask) { - mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); - // try to reclaim it atomically - if (_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) { - mi_atomic_decrement_relaxed(&abandoned_count); - previous->bitmap_idx = bitmap_idx; - previous->count = count; - mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); - mi_segment_t* segment = (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx); - mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0); - //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); - return segment; - } - } - } - } - } - } - } - // no more found - previous->bitmap_idx = 0; - previous->count = 0; - return NULL; -} - - /* ----------------------------------------------------------- Add an arena. ----------------------------------------------------------- */ @@ -896,18 +799,30 @@ static bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id, mi_stats_t* static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept { if (arena_id != NULL) *arena_id = _mi_arena_id_none(); - if (size < MI_ARENA_BLOCK_SIZE) return false; - + if (size < MI_ARENA_BLOCK_SIZE) { + _mi_warning_message("the arena size is too small (memory at %p with size %zu)\n", start, size); + return false; + } if (is_large) { mi_assert_internal(memid.initially_committed && memid.is_pinned); } + if (!_mi_is_aligned(start, MI_SEGMENT_ALIGN)) { + void* const aligned_start = mi_align_up_ptr(start, MI_SEGMENT_ALIGN); + const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; + if (diff >= size || (size - diff) < MI_ARENA_BLOCK_SIZE) { + _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); + return false; + } + start = aligned_start; + size = size - diff; + } const size_t bcount = size / MI_ARENA_BLOCK_SIZE; const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS); const size_t bitmaps = (memid.is_pinned ? 3 : 5); const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t)); mi_memid_t meta_memid; - mi_arena_t* arena = (mi_arena_t*)mi_arena_meta_zalloc(asize, &meta_memid, &_mi_stats_main); // TODO: can we avoid allocating from the OS? + mi_arena_t* arena = (mi_arena_t*)_mi_arena_meta_zalloc(asize, &meta_memid); if (arena == NULL) return false; // already zero'd due to zalloc @@ -924,7 +839,8 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int arena->is_large = is_large; arena->purge_expire = 0; arena->search_idx = 0; - // consequetive bitmaps + mi_lock_init(&arena->abandoned_visit_lock); + // consecutive bitmaps arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap arena->blocks_abandoned = &arena->blocks_inuse[2 * fields]; // just after dirty bitmap arena->blocks_committed = (arena->memid.is_pinned ? NULL : &arena->blocks_inuse[3*fields]); // just after abandoned bitmap @@ -959,11 +875,11 @@ int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exc if (arena_id != NULL) *arena_id = _mi_arena_id_none(); size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block mi_memid_t memid; - void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &memid, &_mi_stats_main); + void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &memid); if (start == NULL) return ENOMEM; const bool is_large = memid.is_pinned; // todo: use separate is_large field? if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) { - _mi_os_free_ex(start, size, commit, memid, &_mi_stats_main); + _mi_os_free_ex(start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); return ENOMEM; } @@ -1011,11 +927,11 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_ return inuse_count; } -void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept { +void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept { size_t max_arenas = mi_atomic_load_relaxed(&mi_arena_count); size_t inuse_total = 0; - size_t abandoned_total = 0; - size_t purge_total = 0; + //size_t abandoned_total = 0; + //size_t purge_total = 0; for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); if (arena == NULL) break; @@ -1026,16 +942,16 @@ void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) if (arena->blocks_committed != NULL) { mi_debug_show_bitmap(" ", "committed blocks", arena->block_count, arena->blocks_committed, arena->field_count); } - if (show_abandoned) { - abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count); - } - if (show_purge && arena->blocks_purge != NULL) { - purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count); - } + //if (show_abandoned) { + // abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count); + //} + //if (show_purge && arena->blocks_purge != NULL) { + // purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count); + //} } if (show_inuse) _mi_verbose_message("total inuse blocks : %zu\n", inuse_total); - if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total); - if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total); + //if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total); + //if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total); } @@ -1059,7 +975,7 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); if (!mi_manage_os_memory_ex2(p, hsize, true, numa_node, exclusive, memid, arena_id)) { - _mi_os_free(p, hsize, memid, &_mi_stats_main); + _mi_os_free(p, hsize, memid); return ENOMEM; } return 0; @@ -1105,4 +1021,3 @@ int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserv if (err==0 && pages_reserved!=NULL) *pages_reserved = pages; return err; } - diff --git a/third-party/mimalloc/src/bitmap.c b/third-party/mimalloc/src/bitmap.c index 4b6be66bcd..084082fb72 100644 --- a/third-party/mimalloc/src/bitmap.c +++ b/third-party/mimalloc/src/bitmap.c @@ -200,7 +200,7 @@ bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t // Try to atomically claim a sequence of `count` bits starting from the field // at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success. // Only needs to consider crossing into the next fields (see `mi_bitmap_try_find_from_claim_across`) -static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats) +static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx) { mi_assert_internal(bitmap_idx != NULL); @@ -260,7 +260,7 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); // claimed! - mi_stat_counter_increase(stats->arena_crossover_count,1); + mi_stat_counter_increase(_mi_stats_main.arena_crossover_count,1); *bitmap_idx = mi_bitmap_index_create(idx, initial_idx); return true; @@ -280,10 +280,10 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit newmap = (map & ~initial_mask); } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); } - mi_stat_counter_increase(stats->arena_rollback_count,1); + mi_stat_counter_increase(_mi_stats_main.arena_rollback_count,1); // retry? (we make a recursive call instead of goto to be able to use const declarations) if (retries <= 2) { - return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx, stats); + return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx); } else { return false; @@ -293,7 +293,7 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit // Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. // Starts at idx, and wraps around to search in all `bitmap_fields` fields. -bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats) { +bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { mi_assert_internal(count > 0); if (count <= 2) { // we don't bother with crossover fields for small counts @@ -313,7 +313,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm } */ // if that fails, then try to claim across fields - if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx, stats)) { + if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) { return true; } } diff --git a/third-party/mimalloc/src/bitmap.h b/third-party/mimalloc/src/bitmap.h index d8316b83f4..f098dd8f13 100644 --- a/third-party/mimalloc/src/bitmap.h +++ b/third-party/mimalloc/src/bitmap.h @@ -35,9 +35,13 @@ typedef mi_bitmap_field_t* mi_bitmap_t; typedef size_t mi_bitmap_index_t; // Create a bit index. +static inline mi_bitmap_index_t mi_bitmap_index_create_ex(size_t idx, size_t bitidx) { + mi_assert_internal(bitidx <= MI_BITMAP_FIELD_BITS); + return (idx*MI_BITMAP_FIELD_BITS) + bitidx; +} static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) { mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS); - return (idx*MI_BITMAP_FIELD_BITS) + bitidx; + return mi_bitmap_index_create_ex(idx,bitidx); } // Create a bit index. @@ -99,7 +103,7 @@ bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t // Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. // Starts at idx, and wraps around to search in all `bitmap_fields` fields. -bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats); +bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); // Set `count` bits at `bitmap_idx` to 0 atomically // Returns `true` if all `count` bits were 1 previously. diff --git a/third-party/mimalloc/src/free.c b/third-party/mimalloc/src/free.c index b9cb634616..f856da77a6 100644 --- a/third-party/mimalloc/src/free.c +++ b/third-party/mimalloc/src/free.c @@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file // add includes help an IDE #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" // _mi_prim_thread_id() #endif @@ -35,7 +34,7 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool if mi_unlikely(mi_check_is_double_free(page, block)) return; mi_check_padding(page, block); if (track_stats) { mi_stat_free(page, block); } - #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN && !MI_GUARDED if (!mi_page_is_huge(page)) { // huge page content may be already decommitted memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); } @@ -54,8 +53,8 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool } // Adjust a block that was allocated aligned, to the actual start of the block in the page. -// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the -// `page_start` and `block_size` fields; however these are constant and the page won't be +// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the +// `page_start` and `block_size` fields; however these are constant and the page won't be // deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently. mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { mi_assert_internal(page!=NULL && p!=NULL); @@ -72,16 +71,30 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { return (mi_block_t*)((uintptr_t)p - adjust); } +// forward declaration for a MI_GUARDED build +#if MI_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration +static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { + if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard(page, block, p); } +} +#else +static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); +} +#endif + // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { MI_UNUSED(segment); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p); + mi_block_check_unguard(page, block, p); mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865) + mi_block_check_unguard(page, block, p); mi_free_block_mt(page, segment, block); } @@ -98,17 +111,17 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms { MI_UNUSED(msg); -#if (MI_DEBUG>0) - if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) { + #if (MI_DEBUG>0) + if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return NULL; } -#endif + #endif mi_segment_t* const segment = _mi_ptr_segment(p); if mi_unlikely(segment==NULL) return segment; -#if (MI_DEBUG>0) + #if (MI_DEBUG>0) if mi_unlikely(!mi_is_in_heap_region(p)) { #if (MI_INTPTR_SIZE == 8 && defined(__linux__)) if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640) @@ -122,13 +135,13 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms } } } -#endif -#if (MI_DEBUG>0 || MI_SECURE>=4) + #endif + #if (MI_DEBUG>0 || MI_SECURE>=4) if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) { _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); return NULL; } -#endif + #endif return segment; } @@ -240,15 +253,17 @@ static void mi_decl_noinline mi_free_block_delayed_mt( mi_page_t* page, mi_block static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block) { // first see if the segment was abandoned and if we can reclaim it into our thread - if (mi_option_is_enabled(mi_option_abandoned_reclaim_on_free) && + if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 && #if MI_HUGE_PAGE_ABANDON segment->page_kind != MI_PAGE_HUGE && #endif - mi_atomic_load_relaxed(&segment->thread_id) == 0) + mi_atomic_load_relaxed(&segment->thread_id) == 0 && // segment is abandoned? + mi_prim_get_default_heap() != (mi_heap_t*)&_mi_heap_empty) // and we did not already exit this thread (without this check, a fresh heap will be initalized (issue #944)) { // the segment is abandoned, try to reclaim it into our heap if (_mi_segment_attempt_reclaim(mi_heap_get_default(), segment)) { - mi_assert_internal(_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + mi_assert_internal(_mi_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + mi_assert_internal(mi_heap_get_default()->tld->segments.subproc == segment->subproc); mi_free(block); // recursively free as now it will be a local free in our heap return; } @@ -299,7 +314,13 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p const size_t size = mi_page_usable_size_of(page, block); const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block; mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); - return (size - adjust); + const size_t aligned_size = (size - adjust); + #if MI_GUARDED + if (mi_block_ptr_is_guarded(block, p)) { + return aligned_size - _mi_os_page_size(); + } + #endif + return aligned_size; } static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { @@ -409,7 +430,7 @@ static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* bloc uintptr_t keys[2]; keys[0] = page->keys[0]; keys[1] = page->keys[1]; - bool ok = ((uint32_t)mi_ptr_encode(page,block,keys) == canary && *delta <= *bsize); + bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize); mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; } @@ -528,3 +549,23 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); } #endif + + +// Remove guard page when building with MI_GUARDED +#if MI_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(p); + mi_assert_internal(mi_block_ptr_is_guarded(block, p)); + mi_assert_internal(mi_page_has_aligned(page)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); + mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); + + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(_mi_page_segment(page)->allow_decommit); + void* gpage = (uint8_t*)block + bsize - psize; + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); +} +#endif diff --git a/third-party/mimalloc/src/heap.c b/third-party/mimalloc/src/heap.c index e498fdb209..f39dfb0ba0 100644 --- a/third-party/mimalloc/src/heap.c +++ b/third-party/mimalloc/src/heap.c @@ -59,7 +59,7 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ MI_UNUSED(pq); mi_assert_internal(mi_page_heap(page) == heap); mi_segment_t* segment = _mi_page_segment(page); - mi_assert_internal(segment->thread_id == heap->thread_id); + mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == heap->thread_id); mi_assert_expensive(_mi_page_is_valid(page)); return true; } @@ -98,7 +98,7 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t if (collect == MI_FORCE) { // note: call before a potential `_mi_page_free` as the segment may be freed if this was the last used page in that segment. mi_segment_t* segment = _mi_page_segment(page); - _mi_segment_collect(segment, true /* force? */, &heap->tld->segments); + _mi_segment_collect(segment, true /* force? */); } if (mi_page_all_free(page)) { // no more used blocks, free the page. @@ -143,6 +143,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) if (force_main) { // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. // if all memory is freed by now, all segments should be freed. + // note: this only collects in the current subprocess _mi_abandoned_reclaim_all(heap, &heap->tld->segments); } @@ -170,9 +171,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) if (force && is_main_thread && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache } - + // collect arenas (this is program wide so don't force purges on abandonment of threads) - _mi_arenas_collect(collect == MI_FORCE /* force purge? */, &heap->tld->stats); + _mi_arenas_collect(collect == MI_FORCE /* force purge? */); } void _mi_heap_collect_abandon(mi_heap_t* heap) { @@ -227,22 +228,28 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); + _mi_heap_guarded_init(heap); // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; } -mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { +mi_decl_nodiscard mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap == NULL) return NULL; - // don't reclaim abandoned pages or otherwise destroy is unsafe - _mi_heap_init(heap, bheap->tld, arena_id, true /* no reclaim */, 0 /* default tag */); + mi_assert(heap_tag >= 0 && heap_tag < 256); + _mi_heap_init(heap, bheap->tld, arena_id, allow_destroy /* no reclaim? */, (uint8_t)heap_tag /* heap tag */); return heap; } +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { + return mi_heap_new_ex(0 /* default heap tag */, false /* don't allow `mi_heap_destroy` */, arena_id); +} + mi_decl_nodiscard mi_heap_t* mi_heap_new(void) { - return mi_heap_new_in_arena(_mi_arena_id_none()); + // don't reclaim abandoned memory or otherwise destroy is unsafe + return mi_heap_new_ex(0 /* default heap tag */, true /* no reclaim */, _mi_arena_id_none()); } bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid) { @@ -375,7 +382,13 @@ void mi_heap_destroy(mi_heap_t* heap) { mi_assert(heap->no_reclaim); mi_assert_expensive(mi_heap_is_valid(heap)); if (heap==NULL || !mi_heap_is_initialized(heap)) return; + #if MI_GUARDED + // _mi_warning_message("'mi_heap_destroy' called but MI_GUARDED is enabled -- using `mi_heap_delete` instead (heap at %p)\n", heap); + mi_heap_delete(heap); + return; + #else if (!heap->no_reclaim) { + _mi_warning_message("'mi_heap_destroy' called but ignored as the heap was not created with 'allow_destroy' (heap at %p)\n", heap); // don't free in case it may contain reclaimed pages mi_heap_delete(heap); } @@ -388,12 +401,14 @@ void mi_heap_destroy(mi_heap_t* heap) { _mi_heap_destroy_pages(heap); mi_heap_free(heap); } + #endif } // forcefully destroy all heaps in the current thread -void _mi_heap_unsafe_destroy_all(void) { - mi_heap_t* bheap = mi_heap_get_backing(); - mi_heap_t* curr = bheap->tld->heaps; +void _mi_heap_unsafe_destroy_all(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + if (heap == NULL) return; + mi_heap_t* curr = heap->tld->heaps; while (curr != NULL) { mi_heap_t* next = curr->next; if (curr->no_reclaim) { @@ -444,6 +459,12 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_heap_reset_pages(from); } +// are two heaps compatible with respect to heap-tag, exclusive arena etc. +static bool mi_heaps_are_compatible(mi_heap_t* heap1, mi_heap_t* heap2) { + return (heap1->tag == heap2->tag && // store same kind of objects + heap1->arena_id == heap2->arena_id); // same arena preference +} + // Safe delete a heap without freeing any still allocated blocks in that heap. void mi_heap_delete(mi_heap_t* heap) { @@ -452,9 +473,10 @@ void mi_heap_delete(mi_heap_t* heap) mi_assert_expensive(mi_heap_is_valid(heap)); if (heap==NULL || !mi_heap_is_initialized(heap)) return; - if (!mi_heap_is_backing(heap)) { + mi_heap_t* bheap = heap->tld->heap_backing; + if (bheap != heap && mi_heaps_are_compatible(bheap,heap)) { // transfer still used pages to the backing heap - mi_heap_absorb(heap->tld->heap_backing, heap); + mi_heap_absorb(bheap, heap); } else { // the backing heap abandons its pages @@ -527,54 +549,97 @@ bool mi_check_owned(const void* p) { enable visiting all blocks of all heaps across threads ----------------------------------------------------------- */ -// Separate struct to keep `mi_page_t` out of the public interface -typedef struct mi_heap_area_ex_s { - mi_heap_area_t area; - mi_page_t* page; -} mi_heap_area_ex_t; +void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); + area->reserved = page->reserved * bsize; + area->committed = page->capacity * bsize; + area->blocks = mi_page_start(page); + area->used = page->used; // number of blocks in use (#553) + area->block_size = ubsize; + area->full_block_size = bsize; + area->heap_tag = page->heap_tag; +} + + +static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) { + mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX); + *shift = MI_INTPTR_BITS - mi_clz(divisor - 1); + *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1); +} + +static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) { + mi_assert_internal(n <= UINT32_MAX); + const uint64_t hi = ((uint64_t)n * magic) >> 32; + return (size_t)((hi + n) >> shift); +} -static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) { - mi_assert(xarea != NULL); - if (xarea==NULL) return true; - const mi_heap_area_t* area = &xarea->area; - mi_page_t* page = xarea->page; +bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) { + mi_assert(area != NULL); + if (area==NULL) return true; mi_assert(page != NULL); if (page == NULL) return true; - _mi_page_free_collect(page,true); + _mi_page_free_collect(page,true); // collect both thread_delayed and local_free mi_assert_internal(page->local_free == NULL); if (page->used == 0) return true; - const size_t bsize = mi_page_block_size(page); - const size_t ubsize = mi_page_usable_block_size(page); // without padding - size_t psize; - uint8_t* pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize); + size_t psize; + uint8_t* const pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize); + mi_heap_t* const heap = mi_page_heap(page); + const size_t bsize = mi_page_block_size(page); + const size_t ubsize = mi_page_usable_block_size(page); // without padding + // optimize page with one block if (page->capacity == 1) { - // optimize page with one block mi_assert_internal(page->used == 1 && page->free == NULL); return visitor(mi_page_heap(page), area, pstart, ubsize, arg); } + mi_assert(bsize <= UINT32_MAX); + + // optimize full pages + if (page->used == page->capacity) { + uint8_t* block = pstart; + for (size_t i = 0; i < page->capacity; i++) { + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } + return true; + } // create a bitmap of free blocks. #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) - uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)]; - memset(free_map, 0, sizeof(free_map)); + uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; + const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS); + memset(free_map, 0, bmapsize * sizeof(intptr_t)); + if (page->capacity % MI_INTPTR_BITS != 0) { + // mark left-over bits at the end as free + size_t shift = (page->capacity % MI_INTPTR_BITS); + uintptr_t mask = (UINTPTR_MAX << shift); + free_map[bmapsize - 1] = mask; + } + + // fast repeated division by the block size + uint64_t magic; + size_t shift; + mi_get_fast_divisor(bsize, &magic, &shift); #if MI_DEBUG>1 size_t free_count = 0; #endif - for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) { #if MI_DEBUG>1 free_count++; #endif mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); size_t offset = (uint8_t*)block - pstart; mi_assert_internal(offset % bsize == 0); - size_t blockidx = offset / bsize; // Todo: avoid division? - mi_assert_internal( blockidx < MI_MAX_BLOCKS); - size_t bitidx = (blockidx / sizeof(uintptr_t)); - size_t bit = blockidx - (bitidx * sizeof(uintptr_t)); + mi_assert_internal(offset <= UINT32_MAX); + size_t blockidx = mi_fast_divide(offset, magic, shift); + mi_assert_internal(blockidx == offset / bsize); + mi_assert_internal(blockidx < MI_MAX_BLOCKS); + size_t bitidx = (blockidx / MI_INTPTR_BITS); + size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); free_map[bitidx] |= ((uintptr_t)1 << bit); } mi_assert_internal(page->capacity == (free_count + page->used)); @@ -583,42 +648,53 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v #if MI_DEBUG>1 size_t used_count = 0; #endif - for (size_t i = 0; i < page->capacity; i++) { - size_t bitidx = (i / sizeof(uintptr_t)); - size_t bit = i - (bitidx * sizeof(uintptr_t)); - uintptr_t m = free_map[bitidx]; - if (bit == 0 && m == UINTPTR_MAX) { - i += (sizeof(uintptr_t) - 1); // skip a run of free blocks + uint8_t* block = pstart; + for (size_t i = 0; i < bmapsize; i++) { + if (free_map[i] == 0) { + // every block is in use + for (size_t j = 0; j < MI_INTPTR_BITS; j++) { + #if MI_DEBUG>1 + used_count++; + #endif + if (!visitor(heap, area, block, ubsize, arg)) return false; + block += bsize; + } } - else if ((m & ((uintptr_t)1 << bit)) == 0) { - #if MI_DEBUG>1 - used_count++; - #endif - uint8_t* block = pstart + (i * bsize); - if (!visitor(mi_page_heap(page), area, block, ubsize, arg)) return false; + else { + // visit the used blocks in the mask + uintptr_t m = ~free_map[i]; + while (m != 0) { + #if MI_DEBUG>1 + used_count++; + #endif + size_t bitidx = mi_ctz(m); + if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false; + m &= m - 1; // clear least significant bit + } + block += bsize * MI_INTPTR_BITS; } } mi_assert_internal(page->used == used_count); return true; } -typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); +// Separate struct to keep `mi_page_t` out of the public interface +typedef struct mi_heap_area_ex_s { + mi_heap_area_t area; + mi_page_t* page; +} mi_heap_area_ex_t; + +typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); + static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { MI_UNUSED(heap); MI_UNUSED(pq); mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; mi_heap_area_ex_t xarea; - const size_t bsize = mi_page_block_size(page); - const size_t ubsize = mi_page_usable_block_size(page); xarea.page = page; - xarea.area.reserved = page->reserved * bsize; - xarea.area.committed = page->capacity * bsize; - xarea.area.blocks = mi_page_start(page); - xarea.area.used = page->used; // number of blocks in use (#553) - xarea.area.block_size = ubsize; - xarea.area.full_block_size = bsize; + _mi_heap_area_init(&xarea.area, page); return fun(heap, &xarea, arg); } @@ -639,7 +715,7 @@ static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; if (args->visit_blocks) { - return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg); + return _mi_heap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg); } else { return true; diff --git a/third-party/mimalloc/src/init.c b/third-party/mimalloc/src/init.c index 6f51ca8923..a2d01db1dd 100644 --- a/third-party/mimalloc/src/init.c +++ b/third-party/mimalloc/src/init.c @@ -88,7 +88,7 @@ const mi_page_t _mi_page_empty = { { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ - { 0, 0 } \ + { 0, 0 }, { 0, 0 } \ MI_STAT_COUNT_END_NULL() @@ -125,19 +125,22 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { NULL, // next false, // can reclaim 0, // tag + #if MI_GUARDED + 0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) + #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; +static mi_decl_cache_align mi_subproc_t mi_subproc_default; + #define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats))) -#define tld_empty_os ((mi_os_tld_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,os))) mi_decl_cache_align static const mi_tld_t tld_empty = { 0, false, NULL, NULL, - { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, tld_empty_stats, tld_empty_os }, // segments - { 0, tld_empty_stats }, // os + { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, &mi_subproc_default, tld_empty_stats }, // segments { MI_STATS_NULL } // stats }; @@ -150,15 +153,14 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; extern mi_heap_t _mi_heap_main; -static mi_tld_t tld_main = { +static mi_decl_cache_align mi_tld_t tld_main = { 0, false, &_mi_heap_main, & _mi_heap_main, - { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, &tld_main.stats, &tld_main.os }, // segments - { 0, &tld_main.stats }, // os + { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, &mi_subproc_default, &tld_main.stats }, // segments { MI_STATS_NULL } // stats }; -mi_heap_t _mi_heap_main = { +mi_decl_cache_align mi_heap_t _mi_heap_main = { &tld_main, MI_ATOMIC_VAR_INIT(NULL), 0, // thread id @@ -171,6 +173,9 @@ mi_heap_t _mi_heap_main = { NULL, // next heap false, // can reclaim 0, // tag + #if MI_GUARDED + 0, 0, 0, 0, 0, + #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -179,6 +184,45 @@ bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. mi_stats_t _mi_stats_main = { MI_STATS_NULL }; +#if MI_GUARDED +mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { + heap->guarded_sample_seed = seed; + if (heap->guarded_sample_seed == 0) { + heap->guarded_sample_seed = _mi_heap_random_next(heap); + } + heap->guarded_sample_rate = sample_rate; + if (heap->guarded_sample_rate >= 1) { + heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; + } + heap->guarded_sample_count = heap->guarded_sample_seed; // count down samples +} + +mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) { + heap->guarded_size_min = min; + heap->guarded_size_max = (min > max ? min : max); +} + +void _mi_heap_guarded_init(mi_heap_t* heap) { + mi_heap_guarded_set_sample_rate(heap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_heap_guarded_set_size_bound(heap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); +} +#else +mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { + MI_UNUSED(heap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +} + +mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) { + MI_UNUSED(heap); MI_UNUSED(min); MI_UNUSED(max); +} +void _mi_heap_guarded_init(mi_heap_t* heap) { + MI_UNUSED(heap); +} +#endif + static void mi_heap_main_init(void) { if (_mi_heap_main.cookie == 0) { @@ -192,6 +236,9 @@ static void mi_heap_main_init(void) { _mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); + mi_lock_init(&mi_subproc_default.abandoned_os_lock); + mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock); + _mi_heap_guarded_init(&_mi_heap_main); } } @@ -201,6 +248,57 @@ mi_heap_t* _mi_heap_main_get(void) { } +/* ----------------------------------------------------------- + Sub process +----------------------------------------------------------- */ + +mi_subproc_id_t mi_subproc_main(void) { + return NULL; +} + +mi_subproc_id_t mi_subproc_new(void) { + mi_memid_t memid = _mi_memid_none(); + mi_subproc_t* subproc = (mi_subproc_t*)_mi_arena_meta_zalloc(sizeof(mi_subproc_t), &memid); + if (subproc == NULL) return NULL; + subproc->memid = memid; + subproc->abandoned_os_list = NULL; + mi_lock_init(&subproc->abandoned_os_lock); + mi_lock_init(&subproc->abandoned_os_visit_lock); + return subproc; +} + +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { + return (subproc_id == NULL ? &mi_subproc_default : (mi_subproc_t*)subproc_id); +} + +void mi_subproc_delete(mi_subproc_id_t subproc_id) { + if (subproc_id == NULL) return; + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + // check if there are no abandoned segments still.. + bool safe_to_delete = false; + mi_lock(&subproc->abandoned_os_lock) { + if (subproc->abandoned_os_list == NULL) { + safe_to_delete = true; + } + } + if (!safe_to_delete) return; + // safe to release + // todo: should we refcount subprocesses? + mi_lock_done(&subproc->abandoned_os_lock); + mi_lock_done(&subproc->abandoned_os_visit_lock); + _mi_arena_meta_free(subproc, subproc->memid, sizeof(mi_subproc_t)); +} + +void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { + mi_heap_t* heap = mi_heap_get_default(); + if (heap == NULL) return; + mi_assert(heap->tld->segments.subproc == &mi_subproc_default); + if (heap->tld->segments.subproc != &mi_subproc_default) return; + heap->tld->segments.subproc = _mi_subproc_from_id(subproc_id); +} + + + /* ----------------------------------------------------------- Initialization and freeing of the thread local heaps ----------------------------------------------------------- */ @@ -218,7 +316,7 @@ typedef struct mi_thread_data_s { // destroy many OS threads, this may causes too much overhead // per thread so we maintain a small cache of recently freed metadata. -#define TD_CACHE_SIZE (16) +#define TD_CACHE_SIZE (32) static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; static mi_thread_data_t* mi_thread_data_zalloc(void) { @@ -239,10 +337,10 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) { // if that fails, allocate as meta data if (td == NULL) { mi_memid_t memid; - td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); + td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid); if (td == NULL) { // if this fails, try once more. (issue #257) - td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); + td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid); if (td == NULL) { // really out of memory _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); @@ -272,7 +370,7 @@ static void mi_thread_data_free( mi_thread_data_t* tdfree ) { } } // if that fails, just free it directly - _mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid, &_mi_stats_main); + _mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid); } void _mi_thread_data_collect(void) { @@ -282,7 +380,7 @@ void _mi_thread_data_collect(void) { if (td != NULL) { td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); if (td != NULL) { - _mi_os_free(td, sizeof(mi_thread_data_t), td->memid, &_mi_stats_main); + _mi_os_free(td, sizeof(mi_thread_data_t), td->memid); } } } @@ -307,7 +405,7 @@ static bool _mi_thread_heap_init(void) { mi_heap_t* heap = &td->heap; _mi_tld_init(tld, heap); // must be before `_mi_heap_init` _mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */); - _mi_heap_set_default_direct(heap); + _mi_heap_set_default_direct(heap); } return false; } @@ -317,9 +415,8 @@ void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { _mi_memcpy_aligned(tld, &tld_empty, sizeof(mi_tld_t)); tld->heap_backing = bheap; tld->heaps = NULL; + tld->segments.subproc = &mi_subproc_default; tld->segments.stats = &tld->stats; - tld->segments.os = &tld->os; - tld->os.stats = &tld->stats; } // Free the thread local default heap (called from `mi_thread_done`) @@ -476,54 +573,15 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) { // -------------------------------------------------------- // Run functions on process init/done, and thread init/done // -------------------------------------------------------- -static void mi_cdecl mi_process_done(void); - static bool os_preloading = true; // true until this module is initialized -static bool mi_redirected = false; // true if malloc redirects to mi_malloc // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. bool mi_decl_noinline _mi_preloading(void) { return os_preloading; } -mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { - return mi_redirected; -} - -// Communicate with the redirection module on Windows -#if defined(_WIN32) && defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) -#ifdef __cplusplus -extern "C" { -#endif -mi_decl_export void _mi_redirect_entry(DWORD reason) { - // called on redirection; careful as this may be called before DllMain - if (reason == DLL_PROCESS_ATTACH) { - mi_redirected = true; - } - else if (reason == DLL_PROCESS_DETACH) { - mi_redirected = false; - } - else if (reason == DLL_THREAD_DETACH) { - mi_thread_done(); - } -} -__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); -__declspec(dllimport) void mi_cdecl mi_allocator_done(void); -#ifdef __cplusplus -} -#endif -#else -static bool mi_allocator_init(const char** message) { - if (message != NULL) *message = NULL; - return true; -} -static void mi_allocator_done(void) { - // nothing to do -} -#endif - -// Called once by the process loader -static void mi_process_load(void) { +// Called once by the process loader from `src/prim/prim.c` +void _mi_process_load(void) { mi_heap_main_init(); #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; @@ -531,17 +589,14 @@ static void mi_process_load(void) { #endif os_preloading = false; mi_assert_internal(_mi_is_main_thread()); - #if !(defined(_WIN32) && defined(MI_SHARED_LIB)) // use Dll process detach (see below) instead of atexit (issue #521) - atexit(&mi_process_done); - #endif _mi_options_init(); mi_process_setup_auto_thread_done(); mi_process_init(); - if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); + if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); // show message from the redirector (if present) const char* msg = NULL; - mi_allocator_init(&msg); + _mi_allocator_init(&msg); if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { _mi_fputs(NULL,NULL,NULL,msg); } @@ -553,12 +608,15 @@ static void mi_process_load(void) { #if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) #include mi_decl_cache_align bool _mi_cpu_has_fsrm = false; +mi_decl_cache_align bool _mi_cpu_has_erms = false; static void mi_detect_cpu_features(void) { - // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + // FSRM for fast short rep movsb/stosb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + // EMRS for fast enhanced rep movsb/stosb support int32_t cpu_info[4]; __cpuid(cpu_info, 7); _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see + _mi_cpu_has_erms = ((cpu_info[2] & (1 << 9)) != 0); // bit 9 of ECX : see } #else static void mi_detect_cpu_features(void) { @@ -619,7 +677,7 @@ void mi_process_init(void) mi_attr_noexcept { } // Called when the process is done (through `at_exit`) -static void mi_cdecl mi_process_done(void) { +void mi_cdecl _mi_process_done(void) { // only shutdown if we were initialized if (!_mi_process_is_initialized) return; // ensure we are called once @@ -627,15 +685,20 @@ static void mi_cdecl mi_process_done(void) { if (process_done) return; process_done = true; + // get the default heap so we don't need to acces thread locals anymore + mi_heap_t* heap = mi_prim_get_default_heap(); // use prim to not initialize any heap + mi_assert_internal(heap != NULL); + // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread _mi_prim_thread_done_auto_done(); + #ifndef MI_SKIP_COLLECT_ON_EXIT #if (MI_DEBUG || !defined(MI_SHARED_LIB)) // free all memory if possible on process exit. This is not needed for a stand-alone process // but should be done if mimalloc is statically linked into another shared library which // is repeatedly loaded/unloaded, see issue #281. - mi_collect(true /* force */ ); + mi_heap_collect(heap, true /* force */ ); #endif #endif @@ -643,72 +706,17 @@ static void mi_cdecl mi_process_done(void) { // since after process_done there might still be other code running that calls `free` (like at_exit routines, // or C-runtime termination code. if (mi_option_is_enabled(mi_option_destroy_on_exit)) { - mi_collect(true /* force */); - _mi_heap_unsafe_destroy_all(); // forcefully release all memory held by all heaps (of this thread only!) - _mi_arena_unsafe_destroy_all(& _mi_heap_main_get()->tld->stats); + mi_heap_collect(heap, true /* force */); + _mi_heap_unsafe_destroy_all(heap); // forcefully release all memory held by all heaps (of this thread only!) + _mi_arena_unsafe_destroy_all(); + _mi_segment_map_unsafe_destroy(); } if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { mi_stats_print(NULL); } - mi_allocator_done(); + _mi_allocator_done(); _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); os_preloading = true; // don't call the C runtime anymore } - - -#if defined(_WIN32) && defined(MI_SHARED_LIB) - // Windows DLL: easy to hook into process_init and thread_done - __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { - MI_UNUSED(reserved); - MI_UNUSED(inst); - if (reason==DLL_PROCESS_ATTACH) { - mi_process_load(); - } - else if (reason==DLL_PROCESS_DETACH) { - mi_process_done(); - } - else if (reason==DLL_THREAD_DETACH) { - if (!mi_is_redirected()) { - mi_thread_done(); - } - } - return TRUE; - } - -#elif defined(_MSC_VER) - // MSVC: use data section magic for static libraries - // See - static int _mi_process_init(void) { - mi_process_load(); - return 0; - } - typedef int(*_mi_crt_callback_t)(void); - #if defined(_M_X64) || defined(_M_ARM64) - __pragma(comment(linker, "/include:" "_mi_msvc_initu")) - #pragma section(".CRT$XIU", long, read) - #else - __pragma(comment(linker, "/include:" "__mi_msvc_initu")) - #endif - #pragma data_seg(".CRT$XIU") - mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init }; - #pragma data_seg() - -#elif defined(__cplusplus) - // C++: use static initialization to detect process start - static bool _mi_process_init(void) { - mi_process_load(); - return (_mi_heap_main.thread_id != 0); - } - static bool mi_initialized = _mi_process_init(); - -#elif defined(__GNUC__) || defined(__clang__) - // GCC,Clang: use the constructor attribute - static void __attribute__((constructor)) _mi_process_init(void) { - mi_process_load(); - } - -#else -#pragma message("define a way to call mi_process_load on your platform") -#endif diff --git a/third-party/mimalloc/src/libc.c b/third-party/mimalloc/src/libc.c index dd6b400737..ce541f1b52 100644 --- a/third-party/mimalloc/src/libc.c +++ b/third-party/mimalloc/src/libc.c @@ -130,7 +130,7 @@ static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, } -static void mi_out_num(uintptr_t x, size_t base, char prefix, char** out, char* end) +static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end) { if (x == 0 || base == 0 || base > 16) { if (prefix != 0) { mi_outc(prefix, out, end); } @@ -206,12 +206,13 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { } else if (c == 'p' || c == 'x' || c == 'u') { // unsigned - uintptr_t x = 0; + uintmax_t x = 0; if (c == 'x' || c == 'u') { if (numtype == 'z') x = va_arg(args, size_t); else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t - else if (numtype == 'L') x = (uintptr_t)va_arg(args, unsigned long long); - else x = va_arg(args, unsigned long); + else if (numtype == 'L') x = va_arg(args, unsigned long long); + else if (numtype == 'l') x = va_arg(args, unsigned long); + else x = va_arg(args, unsigned int); } else if (c == 'p') { x = va_arg(args, uintptr_t); @@ -228,20 +229,21 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { } else if (c == 'i' || c == 'd') { // signed - intptr_t x = 0; + intmax_t x = 0; if (numtype == 'z') x = va_arg(args, intptr_t ); else if (numtype == 't') x = va_arg(args, ptrdiff_t); - else if (numtype == 'L') x = (intptr_t)va_arg(args, long long); - else x = va_arg(args, long); + else if (numtype == 'L') x = va_arg(args, long long); + else if (numtype == 'l') x = va_arg(args, long); + else x = va_arg(args, int); char pre = 0; if (x < 0) { pre = '-'; - if (x > INTPTR_MIN) { x = -x; } + if (x > INTMAX_MIN) { x = -x; } } else if (numplus != 0) { pre = numplus; } - mi_out_num((uintptr_t)x, 10, pre, &out, end); + mi_out_num((uintmax_t)x, 10, pre, &out, end); } else if (c >= ' ' && c <= '~') { // unknown format diff --git a/third-party/mimalloc/src/options.c b/third-party/mimalloc/src/options.c index a62727dd69..d7fa666517 100644 --- a/third-party/mimalloc/src/options.c +++ b/third-party/mimalloc/src/options.c @@ -47,6 +47,62 @@ typedef struct mi_option_desc_s { #define MI_OPTION(opt) mi_option_##opt, #opt, NULL #define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy +// Some options can be set at build time for statically linked libraries +// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) +// +// This is useful if we cannot pass them as environment variables +// (and setting them programmatically would be too late) + +#ifndef MI_DEFAULT_VERBOSE +#define MI_DEFAULT_VERBOSE 0 +#endif + +#ifndef MI_DEFAULT_EAGER_COMMIT +#define MI_DEFAULT_EAGER_COMMIT 1 +#endif + +#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT +#define MI_DEFAULT_ARENA_EAGER_COMMIT 2 +#endif + +// in KiB +#ifndef MI_DEFAULT_ARENA_RESERVE + #if (MI_INTPTR_SIZE>4) + #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L + #else + #define MI_DEFAULT_ARENA_RESERVE 128L*1024L + #endif +#endif + +#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC +#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0 +#endif + +#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES +#if defined(__linux__) && !defined(__ANDROID__) +#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 2 // enabled, but only use transparent huge pages through madvise +#else +#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0 +#endif +#endif + +#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES +#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_OS_MEMORY +#define MI_DEFAULT_RESERVE_OS_MEMORY 0 +#endif + +#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE +#if MI_GUARDED +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000 +#else +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 +#endif +#endif + + static mi_option_desc_t options[_mi_option_last] = { // stable options @@ -56,16 +112,21 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(show_errors) }, #endif { 0, UNINIT, MI_OPTION(show_stats) }, - { 0, UNINIT, MI_OPTION(verbose) }, + { MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) }, - // the following options are experimental and not all combinations make sense. - { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) - { 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + // some of the following options are experimental and not all combinations are allowed. + { MI_DEFAULT_EAGER_COMMIT, + UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) + { MI_DEFAULT_ARENA_EAGER_COMMIT, + UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) - { 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, + UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, + UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) + { MI_DEFAULT_RESERVE_OS_MEMORY, + UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_purge,abandoned_page_reset) }, // reset free page memory when a thread terminates @@ -83,16 +144,24 @@ static mi_option_desc_t options[_mi_option_last] = { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! - #if (MI_INTPTR_SIZE>4) - { 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - #else - { 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit - #endif - { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) + { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, - { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free - { 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { 0, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free + { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. +#if defined(MI_VISIT_ABANDONED) + { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandoned segments; requires taking locks during reclaim. +#else + { 0, UNINIT, MI_OPTION(visit_abandoned) }, +#endif + { 0, UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects + { MI_GiB, UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects + { 0, UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + { MI_DEFAULT_GUARDED_SAMPLE_RATE, + UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000) + { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, + { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable. }; static void mi_option_init(mi_option_desc_t* desc); @@ -102,8 +171,7 @@ static bool mi_option_has_size_in_kib(mi_option_t option) { } void _mi_options_init(void) { - // called on process load; should not be called before the CRT is initialized! - // (e.g. do not call this from process_init as that may run before CRT initialization) + // called on process load mi_add_stderr_output(); // now it safe to use stderr for output for(int i = 0; i < _mi_option_last; i++ ) { mi_option_t option = (mi_option_t)i; @@ -116,8 +184,26 @@ void _mi_options_init(void) { } mi_max_error_count = mi_option_get(mi_option_max_errors); mi_max_warning_count = mi_option_get(mi_option_max_warnings); + #if MI_GUARDED + if (mi_option_get(mi_option_guarded_sample_rate) > 0) { + if (mi_option_is_enabled(mi_option_allow_large_os_pages)) { + mi_option_disable(mi_option_allow_large_os_pages); + _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n"); + } + } + _mi_verbose_message("guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled"); + #endif } +long _mi_option_get_fast(mi_option_t option) { + mi_assert(option >= 0 && option < _mi_option_last); + mi_option_desc_t* desc = &options[option]; + mi_assert(desc->option == option); // index should match the option + //mi_assert(desc->init != UNINIT); + return desc->value; +} + + mi_decl_nodiscard long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); if (option < 0 || option >= _mi_option_last) return 0; @@ -135,7 +221,6 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma } mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { - mi_assert_internal(mi_option_has_size_in_kib(option)); const long x = mi_option_get(option); size_t size = (x < 0 ? 0 : (size_t)x); if (mi_option_has_size_in_kib(option)) { @@ -151,6 +236,13 @@ void mi_option_set(mi_option_t option, long value) { mi_assert(desc->option == option); // index should match the option desc->value = value; desc->init = INITIALIZED; + // ensure min/max range; be careful to not recurse. + if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) { + mi_option_set(mi_option_guarded_max, value); + } + else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) { + mi_option_set(mi_option_guarded_min, value); + } } void mi_option_set_default(mi_option_t option, long value) { @@ -194,7 +286,7 @@ static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { // an output function is registered it is called immediately with // the output up to that point. #ifndef MI_MAX_DELAY_OUTPUT -#define MI_MAX_DELAY_OUTPUT ((size_t)(32*1024)) +#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024)) #endif static char out_buf[MI_MAX_DELAY_OUTPUT+1]; static _Atomic(size_t) out_len; @@ -280,7 +372,7 @@ static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop // (recursively) invoke malloc again to allocate space for the thread local // variables on demand. This is why we use a _mi_preloading test on such // platforms. However, C code generator may move the initial thread local address -// load before the `if` and we therefore split it out in a separate funcion. +// load before the `if` and we therefore split it out in a separate function. static mi_decl_thread bool recurse = false; static mi_decl_noinline bool mi_recurse_enter_prim(void) { @@ -485,7 +577,7 @@ static void mi_option_init(mi_option_desc_t* desc) { char* end = buf; long value = strtol(buf, &end, 10); if (mi_option_has_size_in_kib(desc->option)) { - // this option is interpreted in KiB to prevent overflow of `long` for large allocations + // this option is interpreted in KiB to prevent overflow of `long` for large allocations // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) size_t size = (value < 0 ? 0 : (size_t)value); bool overflow = false; @@ -500,8 +592,7 @@ static void mi_option_init(mi_option_desc_t* desc) { value = (size > LONG_MAX ? LONG_MAX : (long)size); } if (*end == 0) { - desc->value = value; - desc->init = INITIALIZED; + mi_option_set(desc->option, value); } else { // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose. diff --git a/third-party/mimalloc/src/os.c b/third-party/mimalloc/src/os.c index ce104273bf..61c9eebfe9 100644 --- a/third-party/mimalloc/src/os.c +++ b/third-party/mimalloc/src/os.c @@ -9,18 +9,38 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/atomic.h" #include "mimalloc/prim.h" +#define mi_os_stat_increase(stat,amount) _mi_stat_increase(&_mi_stats_main.stat, amount) +#define mi_os_stat_decrease(stat,amount) _mi_stat_decrease(&_mi_stats_main.stat, amount) +#define mi_os_stat_counter_increase(stat,inc) _mi_stat_counter_increase(&_mi_stats_main.stat, inc) /* ----------------------------------------------------------- - Initialization. + Initialization. ----------------------------------------------------------- */ +#ifndef MI_DEFAULT_VIRTUAL_ADDRESS_BITS +#if MI_INTPTR_SIZE < 8 +#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 32 +#else +#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 48 +#endif +#endif + +#ifndef MI_DEFAULT_PHYSICAL_MEMORY +#if MI_INTPTR_SIZE < 8 +#define MI_DEFAULT_PHYSICAL_MEMORY 4*MI_GiB +#else +#define MI_DEFAULT_PHYSICAL_MEMORY 32*MI_GiB +#endif +#endif static mi_os_mem_config_t mi_os_mem_config = { - 4096, // page size - 0, // large page size (usually 2MiB) - 4096, // allocation granularity - true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) - false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) - true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) + 4096, // page size + 0, // large page size (usually 2MiB) + 4096, // allocation granularity + MI_DEFAULT_PHYSICAL_MEMORY, + MI_DEFAULT_VIRTUAL_ADDRESS_BITS, + true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) + false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) + true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) }; bool _mi_os_has_overcommit(void) { @@ -68,17 +88,18 @@ void _mi_os_init(void) { /* ----------------------------------------------------------- Util -------------------------------------------------------------- */ -bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); -bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); +bool _mi_os_decommit(void* addr, size_t size); +bool _mi_os_commit(void* addr, size_t size, bool* is_zero); /* ----------------------------------------------------------- aligned hinting -------------------------------------------------------------- */ -// On 64-bit systems, we can do efficient aligned allocation by using -// the 2TiB to 30TiB area to allocate those. -#if (MI_INTPTR_SIZE >= 8) +// On systems with enough virtual address bits, we can do efficient aligned allocation by using +// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address +// space (64TiB) we use this technique. (but see issue #939) +#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; // Return a MI_SEGMENT_SIZE aligned address that is probably available. @@ -95,6 +116,7 @@ static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; + if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space size = _mi_align_up(size, MI_SEGMENT_SIZE); if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) @@ -122,44 +144,50 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { } #endif - /* ----------------------------------------------------------- Free memory -------------------------------------------------------------- */ -static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats); +static void mi_os_free_huge_os_pages(void* p, size_t size); -static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_stats_t* tld_stats) { - MI_UNUSED(tld_stats); - mi_stats_t* stats = &_mi_stats_main; +static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) { mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) int err = _mi_prim_free(addr, size); if (err != 0) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } - if (still_committed) { _mi_stat_decrease(&stats->committed, size); } - _mi_stat_decrease(&stats->reserved, size); + if (commit_size > 0) { + mi_os_stat_decrease(committed, commit_size); + } + mi_os_stat_decrease(reserved, size); } -void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) { +void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) { if (mi_memkind_is_os(memid.memkind)) { - size_t csize = _mi_os_good_alloc_size(size); + size_t csize = memid.mem.os.size; + if (csize==0) { _mi_os_good_alloc_size(size); } + size_t commit_size = (still_committed ? csize : 0); void* base = addr; // different base? (due to alignment) - if (memid.mem.os.base != NULL) { - mi_assert(memid.mem.os.base <= addr); - mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr); + if (memid.mem.os.base != base) { + mi_assert(memid.mem.os.base <= addr); base = memid.mem.os.base; - csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base); + const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base; + if (memid.mem.os.size==0) { + csize += diff; + } + if (still_committed) { + commit_size -= diff; // the (addr-base) part was already un-committed + } } // free it if (memid.memkind == MI_MEM_OS_HUGE) { mi_assert(memid.is_pinned); - mi_os_free_huge_os_pages(base, csize, tld_stats); + mi_os_free_huge_os_pages(base, csize); } else { - mi_os_prim_free(base, csize, still_committed, tld_stats); + mi_os_prim_free(base, csize, (still_committed ? commit_size : 0)); } } else { @@ -168,8 +196,8 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me } } -void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats) { - _mi_os_free_ex(p, size, true, memid, tld_stats); +void _mi_os_free(void* p, size_t size, mi_memid_t memid) { + _mi_os_free_ex(p, size, true, memid); } @@ -178,7 +206,8 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats) -------------------------------------------------------------- */ // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) { +// Also `hint_addr` is a hint and may be ignored. +static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(is_zero != NULL); mi_assert_internal(is_large != NULL); @@ -187,18 +216,18 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning *is_zero = false; void* p = NULL; - int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, is_zero, &p); + int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p); if (err != 0) { - _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large); + _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large); } - MI_UNUSED(tld_stats); - mi_stats_t* stats = &_mi_stats_main; - mi_stat_counter_increase(stats->mmap_calls, 1); + + + mi_os_stat_counter_increase(mmap_calls, 1); if (p != NULL) { - _mi_stat_increase(&stats->reserved, size); + mi_os_stat_increase(reserved, size); if (commit) { - _mi_stat_increase(&stats->committed, size); + mi_os_stat_increase(committed, size); // seems needed for asan (or `mimalloc-test-api` fails) #ifdef MI_TRACK_ASAN if (*is_zero) { mi_track_mem_defined(p,size); } @@ -209,10 +238,14 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo return p; } +static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) { + return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero); +} + // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. -static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) { +static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base) { mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(is_large != NULL); @@ -222,8 +255,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); - // try first with a hint (this will be aligned directly on Win 10+ or BSD) - void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats); + // try first with a requested alignment hint (this will usually be aligned directly on Win 10+ or BSD) + void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero); if (p == NULL) return NULL; // aligned already? @@ -232,14 +265,16 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit } else { // if not aligned, free it, overallocate, and unmap around it + #if !MI_TRACK_ASAN _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); - mi_os_prim_free(p, size, commit, stats); + #endif + if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); } if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block // over-allocate uncommitted (virtual) memory - p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats); + p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero); if (p == NULL) return NULL; // set p to the aligned part in the full region @@ -250,22 +285,22 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit // explicitly commit only the aligned part if (commit) { - _mi_os_commit(p, size, NULL, stats); + _mi_os_commit(p, size, NULL); } } else { // mmap can free inside an allocation // overallocate... - p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats); + p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero); if (p == NULL) return NULL; - // and selectively unmap parts around the over-allocated area. + // and selectively unmap parts around the over-allocated area. void* aligned_p = mi_align_up_ptr(p, alignment); size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; size_t mid_size = _mi_align_up(size, _mi_os_page_size()); size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); } - if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); } + if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); } + if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); } // we can return the aligned pointer on `mmap` systems p = aligned_p; *base = aligned_p; // since we freed the pre part, `*base == p`. @@ -281,20 +316,20 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit OS API: alloc and alloc_aligned ----------------------------------------------------------- */ -void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats) { +void* _mi_os_alloc(size_t size, mi_memid_t* memid) { *memid = _mi_memid_none(); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); bool os_is_large = false; bool os_is_zero = false; - void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats); + void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero); if (p != NULL) { *memid = _mi_memid_create_os(true, os_is_zero, os_is_large); } return p; } -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats) +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) { MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings *memid = _mi_memid_none(); @@ -305,11 +340,12 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo bool os_is_large = false; bool os_is_zero = false; void* os_base = NULL; - void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, stats ); + void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base ); if (p != NULL) { *memid = _mi_memid_create_os(commit, os_is_zero, os_is_large); memid->mem.os.base = os_base; - memid->mem.os.alignment = alignment; + // memid->mem.os.alignment = alignment; + memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned } return p; } @@ -322,7 +358,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo to use the actual start of the memory region. ----------------------------------------------------------- */ -void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats) { +void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) { mi_assert(offset <= MI_SEGMENT_SIZE); mi_assert(offset <= size); mi_assert((alignment % _mi_os_page_size()) == 0); @@ -330,20 +366,20 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse if (offset > MI_SEGMENT_SIZE) return NULL; if (offset == 0) { // regular aligned allocation - return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid, stats); + return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid); } else { // overallocate to align at an offset const size_t extra = _mi_align_up(offset, alignment) - offset; const size_t oversize = size + extra; - void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, stats); + void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid); if (start == NULL) return NULL; void* const p = (uint8_t*)start + extra; mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); // decommit the overallocation at the start if (commit && extra > _mi_os_page_size()) { - _mi_os_decommit(start, extra, stats); + _mi_os_decommit(start, extra); } return p; } @@ -377,12 +413,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { - MI_UNUSED(tld_stats); - mi_stats_t* stats = &_mi_stats_main; +bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) { if (is_zero != NULL) { *is_zero = false; } - _mi_stat_increase(&stats->committed, size); // use size for precise commit vs. decommit - _mi_stat_counter_increase(&stats->commit_calls, 1); + mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit + mi_os_stat_counter_increase(commit_calls, 1); // page align range size_t csize; @@ -408,11 +442,13 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats return true; } -static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_stats_t* tld_stats) { - MI_UNUSED(tld_stats); - mi_stats_t* stats = &_mi_stats_main; +bool _mi_os_commit(void* addr, size_t size, bool* is_zero) { + return _mi_os_commit_ex(addr, size, is_zero, size); +} + +static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) { mi_assert_internal(needs_recommit!=NULL); - _mi_stat_decrease(&stats->committed, size); + mi_os_stat_decrease(committed, stat_size); // page align size_t csize; @@ -429,9 +465,9 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_ return (err == 0); } -bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { +bool _mi_os_decommit(void* addr, size_t size) { bool needs_recommit; - return mi_os_decommit_ex(addr, size, &needs_recommit, tld_stats); + return mi_os_decommit_ex(addr, size, &needs_recommit, size); } @@ -439,13 +475,13 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { // but may be used later again. This will release physical memory // pages and reduce swapping while keeping the memory committed. // We page align to a conservative area inside the range to reset. -bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { +bool _mi_os_reset(void* addr, size_t size) { // page align conservatively within the range size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) - _mi_stat_increase(&stats->reset, csize); - _mi_stat_counter_increase(&stats->reset_calls, 1); + mi_os_stat_increase(reset, csize); + mi_os_stat_counter_increase(reset_calls, 1); #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN memset(start, 0, csize); // pretend it is eagerly reset @@ -461,22 +497,22 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. -bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats) +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size) { if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? - _mi_stat_counter_increase(&stats->purge_calls, 1); - _mi_stat_increase(&stats->purged, size); + mi_os_stat_counter_increase(purge_calls, 1); + mi_os_stat_increase(purged, size); if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? !_mi_preloading()) // don't decommit during preloading (unsafe) { bool needs_recommit = true; - mi_os_decommit_ex(p, size, &needs_recommit, stats); + mi_os_decommit_ex(p, size, &needs_recommit, stat_size); return needs_recommit; } else { if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed - _mi_os_reset(p, size, stats); + _mi_os_reset(p, size); } return false; // needs no recommit } @@ -484,8 +520,8 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats) // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. -bool _mi_os_purge(void* p, size_t size, mi_stats_t * stats) { - return _mi_os_purge_ex(p, size, true, stats); +bool _mi_os_purge(void* p, size_t size) { + return _mi_os_purge_ex(p, size, true, size); } // Protect a region in memory to be not accessible. @@ -592,15 +628,15 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // no success, issue a warning and break if (p != NULL) { _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); - mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, true, &_mi_stats_main); + mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); } break; } // success, record it page++; // increase before timeout check (see issue #711) - _mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE); - _mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE); + mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE); + mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE); // check for timeout if (max_msecs > 0) { @@ -634,11 +670,11 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // free every huge page in a range individually (as we allocated per page) // note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. -static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats) { +static void mi_os_free_huge_os_pages(void* p, size_t size) { if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { - mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, true, stats); + mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); size -= MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE; } @@ -667,8 +703,7 @@ size_t _mi_os_numa_node_count_get(void) { return count; } -int _mi_os_numa_node_get(mi_os_tld_t* tld) { - MI_UNUSED(tld); +int _mi_os_numa_node_get(void) { size_t numa_count = _mi_os_numa_node_count(); if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 // never more than the node count and >= 0 diff --git a/third-party/mimalloc/src/page-queue.c b/third-party/mimalloc/src/page-queue.c index ceea91ee4d..83b60e931b 100644 --- a/third-party/mimalloc/src/page-queue.c +++ b/third-party/mimalloc/src/page-queue.c @@ -264,8 +264,16 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_ heap->page_count++; } +static void mi_page_queue_move_to_front(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_heap(page) == heap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->first == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push(heap, queue, page); + mi_assert_internal(queue->first == page); +} -static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { +static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_expensive(mi_page_queue_contains(from, page)); mi_assert_expensive(!mi_page_queue_contains(to, page)); @@ -278,6 +286,8 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro (mi_page_is_large_or_huge(page) && mi_page_queue_is_full(to))); mi_heap_t* heap = mi_page_heap(page); + + // delete from `from` if (page->prev != NULL) page->prev->next = page->next; if (page->next != NULL) page->next->prev = page->prev; if (page == from->last) from->last = page->prev; @@ -288,22 +298,59 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro mi_heap_queue_first_update(heap, from); } - page->prev = to->last; - page->next = NULL; - if (to->last != NULL) { - mi_assert_internal(heap == mi_page_heap(to->last)); - to->last->next = page; - to->last = page; + // insert into `to` + if (enqueue_at_end) { + // enqueue at the end + page->prev = to->last; + page->next = NULL; + if (to->last != NULL) { + mi_assert_internal(heap == mi_page_heap(to->last)); + to->last->next = page; + to->last = page; + } + else { + to->first = page; + to->last = page; + mi_heap_queue_first_update(heap, to); + } } else { - to->first = page; - to->last = page; - mi_heap_queue_first_update(heap, to); + if (to->first != NULL) { + // enqueue at 2nd place + mi_assert_internal(heap == mi_page_heap(to->first)); + mi_page_t* next = to->first->next; + page->prev = to->first; + page->next = next; + to->first->next = page; + if (next != NULL) { + next->prev = page; + } + else { + to->last = page; + } + } + else { + // enqueue at the head (singleton list) + page->prev = NULL; + page->next = NULL; + to->first = page; + to->last = page; + mi_heap_queue_first_update(heap, to); + } } mi_page_set_in_full(page, mi_page_queue_is_full(to)); } +static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { + mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page); +} + +static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { + // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`) + mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page); +} + // Only called from `mi_heap_absorb`. size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) { mi_assert_internal(mi_heap_contains_queue(heap,pq)); diff --git a/third-party/mimalloc/src/page.c b/third-party/mimalloc/src/page.c index 871ed21514..6a559af038 100644 --- a/third-party/mimalloc/src/page.c +++ b/third-party/mimalloc/src/page.c @@ -276,7 +276,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size mi_assert_internal(mi_heap_contains_queue(heap, pq)); mi_assert_internal(page_alignment > 0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || block_size == pq->block_size); #endif - mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments, &heap->tld->os); + mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments); if (page == NULL) { // this may be out-of-memory, or an abandoned page was reclaimed (and in our queue) return NULL; @@ -358,7 +358,7 @@ void _mi_page_unfull(mi_page_t* page) { mi_page_set_in_full(page, false); // to get the right queue mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page); mi_page_set_in_full(page, true); - mi_page_queue_enqueue_from(pq, pqfull, page); + mi_page_queue_enqueue_from_full(pq, pqfull, page); } static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { @@ -404,6 +404,28 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { _mi_segment_page_abandon(page,segments_tld); } +// force abandon a page +void _mi_page_force_abandon(mi_page_t* page) { + mi_heap_t* heap = mi_page_heap(page); + // mark page as not using delayed free + _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); + + // ensure this page is no longer in the heap delayed free list + _mi_heap_delayed_free_all(heap); + // We can still access the page meta-info even if it is freed as we ensure + // in `mi_segment_force_abandon` that the segment is not freed (yet) + if (page->capacity == 0) return; // it may have been freed now + + // and now unlink it from the page queue and abandon (or free) + mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page); + if (mi_page_all_free(page)) { + _mi_page_free(page, pq, false); + } + else { + _mi_page_abandon(page, pq); + } +} + // Free a page with no more free blocks void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { @@ -451,6 +473,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { // how to check this efficiently though... // for now, we don't retire if it is the only page left of this size class. mi_page_queue_t* pq = mi_page_queue_of(page); + #if MI_RETIRE_CYCLES > 0 const size_t bsize = mi_page_block_size(page); if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? if (pq->last==page && pq->first==page) { // the only page in the queue? @@ -466,6 +489,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { return; // don't free after all } } + #endif _mi_page_free(page, pq, false); } @@ -712,6 +736,17 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi Find pages with free blocks -------------------------------------------------------------*/ +// search for a best next page to use for at most N pages (often cut short if immediate blocks are available) +#define MI_MAX_CANDIDATE_SEARCH (4) + +// is the page not yet used up to its reserved space? +static bool mi_page_is_expandable(const mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(page->capacity <= page->reserved); + return (page->capacity < page->reserved); +} + + // Find a page with free blocks of `page->block_size`. static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* pq, bool first_try) { @@ -719,39 +754,77 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p #if MI_STAT size_t count = 0; #endif + size_t candidate_count = 0; // we reset this on the first candidate to limit the search + mi_page_t* page_candidate = NULL; // a page with free space mi_page_t* page = pq->first; + while (page != NULL) { mi_page_t* next = page->next; // remember next #if MI_STAT count++; #endif + candidate_count++; - // 0. collect freed blocks by us and other threads + // collect freed blocks by us and other threads _mi_page_free_collect(page, false); - // 1. if the page contains free blocks, we are done - if (mi_page_immediate_available(page)) { - break; // pick this one - } + #if MI_MAX_CANDIDATE_SEARCH > 1 + // search up to N pages for a best candidate - // 2. Try to extend - if (page->capacity < page->reserved) { - mi_page_extend_free(heap, page, heap->tld); - mi_assert_internal(mi_page_immediate_available(page)); - break; + // is the local free list non-empty? + const bool immediate_available = mi_page_immediate_available(page); + + // if the page is completely full, move it to the `mi_pages_full` + // queue so we don't visit long-lived pages too often. + if (!immediate_available && !mi_page_is_expandable(page)) { + mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); + mi_page_to_full(page, pq); + } + else { + // the page has free space, make it a candidate + // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages) + if (page_candidate == NULL) { + page_candidate = page; + candidate_count = 0; + } + // prefer to reuse fuller pages (in the hope the less used page gets freed) + else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page) && !mi_page_is_expandable(page)) { + page_candidate = page; + } + // if we find a non-expandable candidate, or searched for N pages, return with the best candidate + if (immediate_available || candidate_count > MI_MAX_CANDIDATE_SEARCH) { + mi_assert_internal(page_candidate!=NULL); + break; + } + } + #else + // first-fit algorithm + // If the page contains free blocks, we are done + if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) { + break; // pick this one } - // 3. If the page is completely full, move it to the `mi_pages_full` + // If the page is completely full, move it to the `mi_pages_full` // queue so we don't visit long-lived pages too often. mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); + #endif page = next; } // for each page mi_heap_stat_counter_increase(heap, searches, count); + // set the page to the best candidate + if (page_candidate != NULL) { + page = page_candidate; + } + if (page != NULL && !mi_page_immediate_available(page)) { + mi_assert_internal(mi_page_is_expandable(page)); + mi_page_extend_free(heap, page, heap->tld); + } + if (page == NULL) { _mi_heap_collect_retired(heap, false); // perhaps make a page available? page = mi_page_fresh(heap, pq); @@ -761,10 +834,14 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p } } else { - mi_assert(pq->first == page); + // move the page to the front of the queue + mi_page_queue_move_to_front(heap, pq, page); page->retire_expire = 0; + // _mi_heap_collect_retired(heap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this } mi_assert_internal(page == NULL || mi_page_immediate_available(page)); + + return page; } @@ -772,7 +849,9 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p // Find a page with free blocks of `size`. static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { - mi_page_queue_t* pq = mi_page_queue(heap,size); + mi_page_queue_t* pq = mi_page_queue(heap, size); + + // check the first page: we even do this with candidate search or otherwise we re-search every time mi_page_t* page = pq->first; if (page != NULL) { #if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness @@ -791,6 +870,7 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { return page; // fast path } } + return mi_page_queue_find_free_ex(heap, pq, true); } @@ -930,7 +1010,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al mi_assert_internal(mi_page_block_size(page) >= size); // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) - if mi_unlikely(zero && page->block_size == 0) { + if mi_unlikely(zero && mi_page_is_huge(page)) { // note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case. void* p = _mi_page_malloc(heap, page, size); mi_assert_internal(p != NULL); diff --git a/third-party/mimalloc/src/prim/emscripten/prim.c b/third-party/mimalloc/src/prim/emscripten/prim.c index f3797c9e66..82147de799 100644 --- a/third-party/mimalloc/src/prim/emscripten/prim.c +++ b/third-party/mimalloc/src/prim/emscripten/prim.c @@ -71,8 +71,8 @@ int _mi_prim_free(void* addr, size_t size) { extern void* emmalloc_memalign(size_t alignment, size_t size); // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { - MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; // TODO: Track the highest address ever seen; first uses of it are zeroes. // That assumes no one else uses sbrk but us (they could go up, @@ -200,7 +200,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { // Thread init/done //---------------------------------------------------------------- -#ifdef __EMSCRIPTEN_SHARED_MEMORY__ +#if defined(MI_USE_PTHREADS) // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default heap) diff --git a/third-party/mimalloc/src/prim/osx/alloc-override-zone.c b/third-party/mimalloc/src/prim/osx/alloc-override-zone.c index 1515b886b2..d3af170dec 100644 --- a/third-party/mimalloc/src/prim/osx/alloc-override-zone.c +++ b/third-party/mimalloc/src/prim/osx/alloc-override-zone.c @@ -418,9 +418,9 @@ static inline malloc_zone_t* mi_get_default_zone(void) } #if defined(__clang__) -__attribute__((constructor(0))) +__attribute__((constructor(101))) // highest priority #else -__attribute__((constructor)) // seems not supported by g++-11 on the M1 +__attribute__((constructor)) // priority level is not supported by gcc #endif __attribute__((used)) static void _mi_macos_override_malloc(void) { diff --git a/third-party/mimalloc/src/prim/prim.c b/third-party/mimalloc/src/prim/prim.c index 3b7d373642..2002853f28 100644 --- a/third-party/mimalloc/src/prim/prim.c +++ b/third-party/mimalloc/src/prim/prim.c @@ -25,3 +25,52 @@ terms of the MIT license. A copy of the license can be found in the file #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) #endif + +// Generic process initialization +#ifndef MI_PRIM_HAS_PROCESS_ATTACH +#if defined(__GNUC__) || defined(__clang__) + // gcc,clang: use the constructor/destructor attribute + // which for both seem to run before regular constructors/destructors + #if defined(__clang__) + #define mi_attr_constructor __attribute__((constructor(101))) + #define mi_attr_destructor __attribute__((destructor(101))) + #else + #define mi_attr_constructor __attribute__((constructor)) + #define mi_attr_destructor __attribute__((destructor)) + #endif + static void mi_attr_constructor mi_process_attach(void) { + _mi_process_load(); + } + static void mi_attr_destructor mi_process_detach(void) { + _mi_process_done(); + } +#elif defined(__cplusplus) + // C++: use static initialization to detect process start/end + // This is not guaranteed to be first/last but the best we can generally do? + struct mi_init_done_t { + mi_init_done_t() { + _mi_process_load(); + } + ~mi_init_done_t() { + _mi_process_done(); + } + }; + static mi_init_done_t mi_init_done; + #else + #pragma message("define a way to call _mi_process_load/done on your platform") +#endif +#endif + +// Generic allocator init/done callback +#ifndef MI_PRIM_HAS_ALLOCATOR_INIT +bool _mi_is_redirected(void) { + return false; +} +bool _mi_allocator_init(const char** message) { + if (message != NULL) { *message = NULL; } + return true; +} +void _mi_allocator_done(void) { + // nothing to do +} +#endif diff --git a/third-party/mimalloc/src/prim/unix/prim.c b/third-party/mimalloc/src/prim/unix/prim.c index 90a4aac2a1..46869c861e 100644 --- a/third-party/mimalloc/src/prim/unix/prim.c +++ b/third-party/mimalloc/src/prim/unix/prim.c @@ -22,19 +22,18 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" #include // mmap #include // sysconf #include // open, close, read, access -#include +#include // getenv, arc4random_buf #if defined(__linux__) #include - #if defined(MI_NO_THP) - #include - #endif + //#if defined(MI_NO_THP) + #include // THP disable + //#endif #if defined(__GLIBC__) #include // linux mmap flags #else @@ -141,6 +140,12 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) if (psize > 0) { config->page_size = (size_t)psize; config->alloc_granularity = (size_t)psize; + #if defined(_SC_PHYS_PAGES) + long pphys = sysconf(_SC_PHYS_PAGES); + if (pphys > 0 && (size_t)pphys < (SIZE_MAX/(size_t)psize)) { + config->physical_memory = (size_t)pphys * (size_t)psize; + } + #endif } config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this? config->has_overcommit = unix_detect_overcommit(); @@ -183,10 +188,11 @@ int _mi_prim_free(void* addr, size_t size ) { static int unix_madvise(void* addr, size_t size, int advice) { #if defined(__sun) - return madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) + int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) #else - return madvise(addr, size, advice); + int res = madvise(addr, size, advice); #endif + return (res==0 ? 0 : errno); } static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { @@ -242,7 +248,7 @@ static int unix_mmap_fd(void) { #if defined(VM_MAKE_TAG) // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) int os_tag = (int)mi_option_get(mi_option_os_tag); - if (os_tag < 100 || os_tag > 255) { os_tag = 100; } + if (os_tag < 100 || os_tag > 255) { os_tag = 254; } return VM_MAKE_TAG(os_tag); #else return -1; @@ -266,7 +272,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif // huge page allocation - if ((large_only || _mi_os_use_large_page(size, try_alignment)) && allow_large) { + if (allow_large && (large_only || (_mi_os_use_large_page(size, try_alignment) && mi_option_get(mi_option_allow_large_os_pages) == 1))) { static _Atomic(size_t) large_page_try_ok; // = 0; size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { @@ -287,7 +293,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #endif #ifdef MAP_HUGE_1GB static bool mi_huge_pages_available = true; - if ((size % MI_GiB) == 0 && mi_huge_pages_available) { + if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) { lflags |= MAP_HUGE_1GB; } else @@ -307,7 +313,9 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #ifdef MAP_HUGE_1GB if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) { mi_huge_pages_available = false; // don't try huge 1GiB pages again - _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); + if (large_only) { + _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno); + } lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd); } @@ -333,7 +341,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec // when large OS pages are enabled for mimalloc, we call `madvise` anyways. if (allow_large && _mi_os_use_large_page(size, try_alignment)) { if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { - *is_large = true; // possibly + // *is_large = true; // possibly }; } #elif defined(__sun) @@ -342,7 +350,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec cmd.mha_pagesize = _mi_os_large_page_size(); cmd.mha_cmd = MHA_MAPSIZE_VA; if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - *is_large = true; + // *is_large = true; // possibly } } #endif @@ -352,14 +360,14 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); *is_zero = true; int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); - *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large); return (*addr != NULL ? 0 : errno); } @@ -761,7 +769,7 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { #include bool _mi_prim_random_buf(void* buf, size_t buf_len) { - // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf // may fail silently on macOS. See PR #390, and return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); } diff --git a/third-party/mimalloc/src/prim/wasi/prim.c b/third-party/mimalloc/src/prim/wasi/prim.c index e95f67f587..e1e7de5efd 100644 --- a/third-party/mimalloc/src/prim/wasi/prim.c +++ b/third-party/mimalloc/src/prim/wasi/prim.c @@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" #include // fputs @@ -22,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file void _mi_prim_mem_init( mi_os_mem_config_t* config ) { config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB config->alloc_granularity = 16; - config->has_overcommit = false; + config->has_overcommit = false; config->has_partial_free = false; config->has_virtual_reserve = false; } @@ -120,8 +119,8 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { - MI_UNUSED(allow_large); MI_UNUSED(commit); +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; *is_zero = false; *addr = mi_prim_mem_grow(size, try_alignment); @@ -134,7 +133,7 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la //--------------------------------------------- int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { - MI_UNUSED(addr); MI_UNUSED(size); + MI_UNUSED(addr); MI_UNUSED(size); *is_zero = false; return 0; } @@ -199,9 +198,9 @@ mi_msecs_t _mi_prim_clock_now(void) { // low resolution timer mi_msecs_t _mi_prim_clock_now(void) { #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) - return (mi_msecs_t)clock(); + return (mi_msecs_t)clock(); #elif (CLOCKS_PER_SEC < 1000) - return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); + return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); #else return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); #endif diff --git a/third-party/mimalloc/src/prim/windows/prim.c b/third-party/mimalloc/src/prim/windows/prim.c index 5074ad4cbd..745224c271 100644 --- a/third-party/mimalloc/src/prim/windows/prim.c +++ b/third-party/mimalloc/src/prim/windows/prim.c @@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/atomic.h" #include "mimalloc/prim.h" #include // fputs, stderr @@ -51,7 +50,7 @@ typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T* static PVirtualAlloc2 pVirtualAlloc2 = NULL; static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL; -// Similarly, GetNumaProcesorNodeEx is only supported since Windows 7 +// Similarly, GetNumaProcessorNodeEx is only supported since Windows 7 typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER; typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber); @@ -119,6 +118,18 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) GetSystemInfo(&si); if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; } if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; } + // get virtual address bits + if ((uintptr_t)si.lpMaximumApplicationAddress > 0) { + const size_t vbits = MI_INTPTR_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress); + config->virtual_address_bits = vbits; + } + // get physical memory + ULONGLONG memInKiB = 0; + if (GetPhysicallyInstalledSystemMemory(&memInKiB)) { + if (memInKiB > 0 && memInKiB < (SIZE_MAX / MI_KiB)) { + config->physical_memory = memInKiB * MI_KiB; + } + } // get the VirtualAlloc2 function HINSTANCE hDll; hDll = LoadLibrary(TEXT("kernelbase.dll")); @@ -192,7 +203,7 @@ static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_ali } #endif // on modern Windows try use VirtualAlloc2 for aligned allocation - if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; reqs.Alignment = try_alignment; MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; @@ -231,7 +242,7 @@ static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignmen else if (max_retry_msecs > 0 && (try_alignment <= 2*MI_SEGMENT_ALIGN) && (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 && win_is_out_of_memory_error(GetLastError())) { - // if committing regular memory and being out-of-memory, + // if committing regular memory and being out-of-memory, // keep trying for a bit in case memory frees up after all. See issue #894 _mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags); long sleep_msecs = tries*40; // increasing waits @@ -280,14 +291,14 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW return p; } -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); *is_zero = true; int flags = MEM_RESERVE; if (commit) { flags |= MEM_COMMIT; } - *addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large); return (*addr != NULL ? 0 : (int)GetLastError()); } @@ -316,7 +327,7 @@ int _mi_prim_commit(void* addr, size_t size, bool* is_zero) { return 0; } -int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { +int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) { BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT); *needs_recommit = true; // for safety, assume always decommitted even in the case of an error. return (ok ? 0 : (int)GetLastError()); @@ -468,7 +479,6 @@ mi_msecs_t _mi_prim_clock_now(void) { // Process Info //---------------------------------------------------------------- -#include #include static mi_msecs_t filetime_msecs(const FILETIME* ftime) { @@ -491,7 +501,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); pinfo->utime = filetime_msecs(&ut); pinfo->stime = filetime_msecs(&st); - + // load psapi on demand if (pGetProcessMemoryInfo == NULL) { HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); @@ -501,11 +511,10 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) } // get process info - PROCESS_MEMORY_COUNTERS info; - memset(&info, 0, sizeof(info)); + PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); if (pGetProcessMemoryInfo != NULL) { pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); - } + } pinfo->current_rss = (size_t)info.WorkingSetSize; pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; pinfo->current_commit = (size_t)info.PagefileUsage; @@ -517,7 +526,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) // Output //---------------------------------------------------------------- -void _mi_prim_out_stderr( const char* msg ) +void _mi_prim_out_stderr( const char* msg ) { // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. @@ -564,7 +573,6 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { } - //---------------------------------------------------------------- // Random //---------------------------------------------------------------- @@ -600,64 +608,210 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { } if (pBCryptGenRandom == NULL) return false; } - return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); + return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0); } #endif // MI_USE_RTLGENRANDOM + + //---------------------------------------------------------------- -// Thread init/done +// Process & Thread Init/Done //---------------------------------------------------------------- -#if !defined(MI_SHARED_LIB) +static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { + MI_UNUSED(reserved); + MI_UNUSED(module); + #if MI_TLS_SLOT >= 2 + if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) { + _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); + } + #endif + if (reason==DLL_PROCESS_ATTACH) { + _mi_process_load(); + } + else if (reason==DLL_PROCESS_DETACH) { + _mi_process_done(); + } + else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) { + _mi_thread_done(NULL); + } +} -// use thread local storage keys to detect thread ending -// note: another design could be to use special linker sections (see issue #869) -#include -#if (_WIN32_WINNT < 0x600) // before Windows Vista -WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); -WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); -WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); -WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); -#endif -static DWORD mi_fls_key = (DWORD)(-1); +#if defined(MI_SHARED_LIB) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 -static void NTAPI mi_fls_done(PVOID value) { - mi_heap_t* heap = (mi_heap_t*)value; - if (heap != NULL) { - _mi_thread_done(heap); - FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 + // Windows DLL: easy to hook into process_init and thread_done + __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { + mi_win_main((PVOID)inst,reason,reserved); + return TRUE; } -} -void _mi_prim_thread_init_auto_done(void) { - mi_fls_key = FlsAlloc(&mi_fls_done); -} + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) { } + void _mi_prim_thread_done_auto_done(void) { } + void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); + } -void _mi_prim_thread_done_auto_done(void) { - // call thread-done on all threads (except the main thread) to prevent - // dangling callback pointer if statically linked with a DLL; Issue #208 - FlsFree(mi_fls_key); -} +#elif !defined(MI_WIN_USE_FLS) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 -void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { - mi_assert_internal(mi_fls_key != (DWORD)(-1)); - FlsSetValue(mi_fls_key, heap); -} + static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) { + mi_win_main(module, reason, reserved); + } + } + static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) { + mi_win_main(module, reason, reserved); + } + } -#else + // Set up TLS callbacks in a statically linked library by using special data sections. + // See + // We use 2 entries to ensure we call attach events before constructors + // are called, and detach events after destructors are called. + #if defined(__cplusplus) + extern "C" { + #endif -// Dll; nothing to do as in that case thread_done is handled through the DLL_THREAD_DETACH event. + #if defined(_WIN64) + #pragma comment(linker, "/INCLUDE:_tls_used") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma const_seg(".CRT$XLB") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; + #pragma const_seg() + #pragma const_seg(".CRT$XLY") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; + #pragma const_seg() + #else + #pragma comment(linker, "/INCLUDE:__tls_used") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma data_seg(".CRT$XLB") + PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; + #pragma data_seg() + #pragma data_seg(".CRT$XLY") + PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; + #pragma data_seg() + #endif -void _mi_prim_thread_init_auto_done(void) { -} + #if defined(__cplusplus) + } + #endif -void _mi_prim_thread_done_auto_done(void) { -} + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) { } + void _mi_prim_thread_done_auto_done(void) { } + void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); + } -void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { - MI_UNUSED(heap); -} +#else // deprecated: statically linked, use fiber api + #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`) + // MSVC: use data section magic for static libraries + // See + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + + static int mi_process_attach(void) { + mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); + atexit(&_mi_process_done); + return 0; + } + typedef int(*mi_crt_callback_t)(void); + #if defined(_WIN64) + #pragma comment(linker, "/INCLUDE:_mi_tls_callback") + #pragma section(".CRT$XIU", long, read) + #else + #pragma comment(linker, "/INCLUDE:__mi_tls_callback") + #endif + #pragma data_seg(".CRT$XIU") + mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach }; + #pragma data_seg() + #endif + + // use the fiber api for calling `_mi_thread_done`. + #include + #if (_WIN32_WINNT < 0x600) // before Windows Vista + WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); + WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); + WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); + WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); + #endif + + static DWORD mi_fls_key = (DWORD)(-1); + + static void NTAPI mi_fls_done(PVOID value) { + mi_heap_t* heap = (mi_heap_t*)value; + if (heap != NULL) { + _mi_thread_done(heap); + FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 + } + } + + void _mi_prim_thread_init_auto_done(void) { + mi_fls_key = FlsAlloc(&mi_fls_done); + } + + void _mi_prim_thread_done_auto_done(void) { + // call thread-done on all threads (except the main thread) to prevent + // dangling callback pointer if statically linked with a DLL; Issue #208 + FlsFree(mi_fls_key); + } + + void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + mi_assert_internal(mi_fls_key != (DWORD)(-1)); + FlsSetValue(mi_fls_key, heap); + } +#endif + +// ---------------------------------------------------- +// Communicate with the redirection module on Windows +// ---------------------------------------------------- +#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) + #define MI_PRIM_HAS_ALLOCATOR_INIT 1 + + static bool mi_redirected = false; // true if malloc redirects to mi_malloc + + bool _mi_is_redirected(void) { + return mi_redirected; + } + + #ifdef __cplusplus + extern "C" { + #endif + mi_decl_export void _mi_redirect_entry(DWORD reason) { + // called on redirection; careful as this may be called before DllMain + #if MI_TLS_SLOT >= 2 + if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) { + _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); + } + #endif + if (reason == DLL_PROCESS_ATTACH) { + mi_redirected = true; + } + else if (reason == DLL_PROCESS_DETACH) { + mi_redirected = false; + } + else if (reason == DLL_THREAD_DETACH) { + _mi_thread_done(NULL); + } + } + __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); + __declspec(dllimport) void mi_cdecl mi_allocator_done(void); + #ifdef __cplusplus + } + #endif + bool _mi_allocator_init(const char** message) { + return mi_allocator_init(message); + } + void _mi_allocator_done(void) { + mi_allocator_done(); + } #endif diff --git a/third-party/mimalloc/src/segment-map.c b/third-party/mimalloc/src/segment-map.c index 1efb1e2360..5809342cf7 100644 --- a/third-party/mimalloc/src/segment-map.c +++ b/third-party/mimalloc/src/segment-map.c @@ -16,140 +16,121 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "mimalloc/atomic.h" -#if (MI_INTPTR_SIZE>=8) && MI_TRACK_ASAN -#define MI_MAX_ADDRESS ((size_t)140 << 40) // 140TB (see issue #881) -#elif (MI_INTPTR_SIZE >= 8) -#define MI_MAX_ADDRESS ((size_t)40 << 40) // 40TB (to include huge page areas) +// Reduce total address space to reduce .bss (due to the `mi_segment_map`) +#if (MI_INTPTR_SIZE > 4) && MI_TRACK_ASAN +#define MI_SEGMENT_MAP_MAX_ADDRESS (128*1024ULL*MI_GiB) // 128 TiB (see issue #881) +#elif (MI_INTPTR_SIZE > 4) +#define MI_SEGMENT_MAP_MAX_ADDRESS (48*1024ULL*MI_GiB) // 48 TiB #else -#define MI_MAX_ADDRESS ((size_t)2 << 30) // 2Gb +#define MI_SEGMENT_MAP_MAX_ADDRESS (UINT32_MAX) #endif -#define MI_SEGMENT_MAP_BITS (MI_MAX_ADDRESS / MI_SEGMENT_SIZE) -#define MI_SEGMENT_MAP_SIZE (MI_SEGMENT_MAP_BITS / 8) -#define MI_SEGMENT_MAP_WSIZE (MI_SEGMENT_MAP_SIZE / MI_INTPTR_SIZE) +#define MI_SEGMENT_MAP_PART_SIZE (MI_INTPTR_SIZE*MI_KiB - 128) // 128 > sizeof(mi_memid_t) ! +#define MI_SEGMENT_MAP_PART_BITS (8*MI_SEGMENT_MAP_PART_SIZE) +#define MI_SEGMENT_MAP_PART_ENTRIES (MI_SEGMENT_MAP_PART_SIZE / MI_INTPTR_SIZE) +#define MI_SEGMENT_MAP_PART_BIT_SPAN (MI_SEGMENT_ALIGN) +#define MI_SEGMENT_MAP_PART_SPAN (MI_SEGMENT_MAP_PART_BITS * MI_SEGMENT_MAP_PART_BIT_SPAN) +#define MI_SEGMENT_MAP_MAX_PARTS ((MI_SEGMENT_MAP_MAX_ADDRESS / MI_SEGMENT_MAP_PART_SPAN) + 1) -static _Atomic(uintptr_t) mi_segment_map[MI_SEGMENT_MAP_WSIZE + 1]; // 2KiB per TB with 64MiB segments +// A part of the segment map. +typedef struct mi_segmap_part_s { + mi_memid_t memid; + _Atomic(uintptr_t) map[MI_SEGMENT_MAP_PART_ENTRIES]; +} mi_segmap_part_t; -static size_t mi_segment_map_index_of(const mi_segment_t* segment, size_t* bitidx) { +// Allocate parts on-demand to reduce .bss footprint +static _Atomic(mi_segmap_part_t*) mi_segment_map[MI_SEGMENT_MAP_MAX_PARTS]; // = { NULL, .. } + +static mi_segmap_part_t* mi_segment_map_index_of(const mi_segment_t* segment, bool create_on_demand, size_t* idx, size_t* bitidx) { // note: segment can be invalid or NULL. mi_assert_internal(_mi_ptr_segment(segment + 1) == segment); // is it aligned on MI_SEGMENT_SIZE? - if ((uintptr_t)segment >= MI_MAX_ADDRESS) { - *bitidx = 0; - return MI_SEGMENT_MAP_WSIZE; - } - else { - const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_SIZE; - *bitidx = segindex % MI_INTPTR_BITS; - const size_t mapindex = segindex / MI_INTPTR_BITS; - mi_assert_internal(mapindex < MI_SEGMENT_MAP_WSIZE); - return mapindex; + *idx = 0; + *bitidx = 0; + if ((uintptr_t)segment >= MI_SEGMENT_MAP_MAX_ADDRESS) return NULL; + const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_MAP_PART_SPAN; + if (segindex >= MI_SEGMENT_MAP_MAX_PARTS) return NULL; + mi_segmap_part_t* part = mi_atomic_load_ptr_relaxed(mi_segmap_part_t, &mi_segment_map[segindex]); + + // allocate on demand to reduce .bss footprint + if (part == NULL) { + if (!create_on_demand) return NULL; + mi_memid_t memid; + part = (mi_segmap_part_t*)_mi_os_alloc(sizeof(mi_segmap_part_t), &memid); + if (part == NULL) return NULL; + part->memid = memid; + mi_segmap_part_t* expected = NULL; + if (!mi_atomic_cas_ptr_strong_release(mi_segmap_part_t, &mi_segment_map[segindex], &expected, part)) { + _mi_os_free(part, sizeof(mi_segmap_part_t), memid); + part = expected; + if (part == NULL) return NULL; + } } + mi_assert(part != NULL); + const uintptr_t offset = ((uintptr_t)segment) % MI_SEGMENT_MAP_PART_SPAN; + const uintptr_t bitofs = offset / MI_SEGMENT_MAP_PART_BIT_SPAN; + *idx = bitofs / MI_INTPTR_BITS; + *bitidx = bitofs % MI_INTPTR_BITS; + return part; } void _mi_segment_map_allocated_at(const mi_segment_t* segment) { + if (segment->memid.memkind == MI_MEM_ARENA) return; // we lookup segments first in the arena's and don't need the segment map + size_t index; size_t bitidx; - size_t index = mi_segment_map_index_of(segment, &bitidx); - mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); - if (index==MI_SEGMENT_MAP_WSIZE) return; - uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + mi_segmap_part_t* part = mi_segment_map_index_of(segment, true /* alloc map if needed */, &index, &bitidx); + if (part == NULL) return; // outside our address range.. + uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); uintptr_t newmask; do { newmask = (mask | ((uintptr_t)1 << bitidx)); - } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); + } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask)); } void _mi_segment_map_freed_at(const mi_segment_t* segment) { + if (segment->memid.memkind == MI_MEM_ARENA) return; + size_t index; size_t bitidx; - size_t index = mi_segment_map_index_of(segment, &bitidx); - mi_assert_internal(index <= MI_SEGMENT_MAP_WSIZE); - if (index == MI_SEGMENT_MAP_WSIZE) return; - uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* don't alloc if not present */, &index, &bitidx); + if (part == NULL) return; // outside our address range.. + uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); uintptr_t newmask; do { newmask = (mask & ~((uintptr_t)1 << bitidx)); - } while (!mi_atomic_cas_weak_release(&mi_segment_map[index], &mask, newmask)); + } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask)); } // Determine the segment belonging to a pointer or NULL if it is not in a valid segment. static mi_segment_t* _mi_segment_of(const void* p) { if (p == NULL) return NULL; mi_segment_t* segment = _mi_ptr_segment(p); // segment can be NULL + size_t index; size_t bitidx; - size_t index = mi_segment_map_index_of(segment, &bitidx); - // fast path: for any pointer to valid small/medium/large object or first MI_SEGMENT_SIZE in huge - const uintptr_t mask = mi_atomic_load_relaxed(&mi_segment_map[index]); + mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* dont alloc if not present */, &index, &bitidx); + if (part == NULL) return NULL; + const uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); if mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0) { + bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); + mi_assert_internal(cookie_ok); MI_UNUSED(cookie_ok); return segment; // yes, allocated by us } - if (index==MI_SEGMENT_MAP_WSIZE) return NULL; - - // TODO: maintain max/min allocated range for efficiency for more efficient rejection of invalid pointers? - - // search downwards for the first segment in case it is an interior pointer - // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps trough - // valid huge objects - // note: we could maintain a lowest index to speed up the path for invalid pointers? - size_t lobitidx; - size_t loindex; - uintptr_t lobits = mask & (((uintptr_t)1 << bitidx) - 1); - if (lobits != 0) { - loindex = index; - lobitidx = mi_bsr(lobits); // lobits != 0 - } - else if (index == 0) { - return NULL; - } - else { - mi_assert_internal(index > 0); - uintptr_t lomask = mask; - loindex = index; - do { - loindex--; - lomask = mi_atomic_load_relaxed(&mi_segment_map[loindex]); - } while (lomask != 0 && loindex > 0); - if (lomask == 0) return NULL; - lobitidx = mi_bsr(lomask); // lomask != 0 - } - mi_assert_internal(loindex < MI_SEGMENT_MAP_WSIZE); - // take difference as the addresses could be larger than the MAX_ADDRESS space. - size_t diff = (((index - loindex) * (8*MI_INTPTR_SIZE)) + bitidx - lobitidx) * MI_SEGMENT_SIZE; - segment = (mi_segment_t*)((uint8_t*)segment - diff); - - if (segment == NULL) return NULL; - mi_assert_internal((void*)segment < p); - bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie); - mi_assert_internal(cookie_ok); - if mi_unlikely(!cookie_ok) return NULL; - if (((uint8_t*)segment + mi_segment_size(segment)) <= (uint8_t*)p) return NULL; // outside the range - mi_assert_internal(p >= (void*)segment && (uint8_t*)p < (uint8_t*)segment + mi_segment_size(segment)); - return segment; + return NULL; } // Is this a valid pointer in our heap? -static bool mi_is_valid_pointer(const void* p) { - return ((_mi_segment_of(p) != NULL) || (_mi_arena_contains(p))); +static bool mi_is_valid_pointer(const void* p) { + // first check if it is in an arena, then check if it is OS allocated + return (_mi_arena_contains(p) || _mi_segment_of(p) != NULL); } mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { return mi_is_valid_pointer(p); } -/* -// Return the full segment range belonging to a pointer -static void* mi_segment_range_of(const void* p, size_t* size) { - mi_segment_t* segment = _mi_segment_of(p); - if (segment == NULL) { - if (size != NULL) *size = 0; - return NULL; - } - else { - if (size != NULL) *size = segment->segment_size; - return segment; +void _mi_segment_map_unsafe_destroy(void) { + for (size_t i = 0; i < MI_SEGMENT_MAP_MAX_PARTS; i++) { + mi_segmap_part_t* part = mi_atomic_exchange_ptr_relaxed(mi_segmap_part_t, &mi_segment_map[i], NULL); + if (part != NULL) { + _mi_os_free(part, sizeof(mi_segmap_part_t), part->memid); + } } - mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); - mi_assert_internal(page == NULL || (mi_segment_page_size(_mi_page_segment(page)) - (MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= block_size); - mi_reset_delayed(tld); - mi_assert_internal(page == NULL || mi_page_not_in_queue(page, tld)); - return page; } -*/ diff --git a/third-party/mimalloc/src/segment.c b/third-party/mimalloc/src/segment.c index 4e4dcb80ee..2c0b9ec31d 100644 --- a/third-party/mimalloc/src/segment.c +++ b/third-party/mimalloc/src/segment.c @@ -17,7 +17,7 @@ terms of the MIT license. A copy of the license can be found in the file // ------------------------------------------------------------------- -static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats); +static void mi_segment_try_purge(mi_segment_t* segment, bool force); // ------------------------------------------------------------------- @@ -150,6 +150,23 @@ size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx) { /* -------------------------------------------------------------------------------- Segment allocation + We allocate pages inside bigger "segments" (32 MiB on 64-bit). This is to avoid + splitting VMA's on Linux and reduce fragmentation on other OS's. + Each thread owns its own segments. + + Currently we have: + - small pages (64KiB) + - medium pages (512KiB) + - large pages (4MiB), + - huge segments have 1 page in one segment that can be larger than `MI_SEGMENT_SIZE`. + it is used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or with alignment `> MI_BLOCK_ALIGNMENT_MAX`. + + The memory for a segment is usually committed on demand. + (i.e. we are careful to not touch the memory until we actually allocate a block there) + + If a thread ends, it "abandons" pages that still contain live blocks. + Such segments are abandoned and these can be reclaimed by still running threads, + (much like work-stealing). -------------------------------------------------------------------------------- */ @@ -332,6 +349,9 @@ static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, c if (block_size <= 64) { start_offset += 3*block_size; } else if (block_size <= 512) { start_offset += block_size; } } + start_offset = _mi_align_up(start_offset, MI_MAX_ALIGN_SIZE); + mi_assert_internal(_mi_is_aligned(pstart + start_offset, MI_MAX_ALIGN_SIZE)); + mi_assert_internal(block_size == 0 || block_size > MI_MAX_ALIGN_GUARANTEE || _mi_is_aligned(pstart + start_offset,block_size)); if (page_size != NULL) { *page_size = psize - start_offset; } return (pstart + start_offset); } @@ -407,8 +427,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { const size_t size = mi_segment_size(segment); const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); - _mi_abandoned_await_readers(); // wait until safe to free - _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid, tld->stats); + _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid); } /* ----------------------------------------------------------- @@ -465,7 +484,7 @@ static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uin mi_commit_mask_create(bitidx, bitcount, cm); } -static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { +static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size) { mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); // commit liberal @@ -481,7 +500,7 @@ static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi mi_commit_mask_t cmask; mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap - if (!_mi_os_commit(start, full_size, &is_zero, stats)) return false; + if (!_mi_os_commit(start, full_size, &is_zero)) return false; mi_commit_mask_set(&segment->commit_mask, &mask); } @@ -495,15 +514,15 @@ static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi return true; } -static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { +static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size) { mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->purge_mask)) return true; // fully committed mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); - return mi_segment_commit(segment, p, size, stats); + return mi_segment_commit(segment, p, size); } -static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { +static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size) { mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); if (!segment->allow_purge) return true; @@ -518,7 +537,7 @@ static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_ // purging mi_assert_internal((void*)start != (void*)segment); mi_assert_internal(segment->allow_decommit); - const bool decommitted = _mi_os_purge(start, full_size, stats); // reset or decommit + const bool decommitted = _mi_os_purge(start, full_size); // reset or decommit if (decommitted) { mi_commit_mask_t cmask; mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); @@ -532,11 +551,11 @@ static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_ return true; } -static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { +static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size) { if (!segment->allow_purge) return; if (mi_option_get(mi_option_purge_delay) == 0) { - mi_segment_purge(segment, p, size, stats); + mi_segment_purge(segment, p, size); } else { // register for future purge in the purge mask @@ -559,7 +578,7 @@ static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t else if (segment->purge_expire <= now) { // previous purge mask already expired if (segment->purge_expire + mi_option_get(mi_option_purge_extend_delay) <= now) { - mi_segment_try_purge(segment, true, stats); + mi_segment_try_purge(segment, true); } else { segment->purge_expire = now + mi_option_get(mi_option_purge_extend_delay); // (mi_option_get(mi_option_purge_delay) / 8); // wait a tiny bit longer in case there is a series of free's @@ -572,7 +591,7 @@ static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t } } -static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) { +static void mi_segment_try_purge(mi_segment_t* segment, bool force) { if (!segment->allow_purge || segment->purge_expire == 0 || mi_commit_mask_is_empty(&segment->purge_mask)) return; mi_msecs_t now = _mi_clock_now(); if (!force && now < segment->purge_expire) return; @@ -588,7 +607,7 @@ static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* if (count > 0) { uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE); size_t size = count * MI_COMMIT_SIZE; - mi_segment_purge(segment, p, size, stats); + mi_segment_purge(segment, p, size); } } mi_commit_mask_foreach_end() @@ -597,8 +616,8 @@ static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* // called from `mi_heap_collect_ex` // this can be called per-page so it is important that try_purge has fast exit path -void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { - mi_segment_try_purge(segment, force, tld->stats); +void _mi_segment_collect(mi_segment_t* segment, bool force) { + mi_segment_try_purge(segment, force); } /* ----------------------------------------------------------- @@ -633,7 +652,7 @@ static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size // perhaps decommit if (allow_purge) { - mi_segment_schedule_purge(segment, mi_slice_start(slice), slice_count * MI_SEGMENT_SLICE_SIZE, tld->stats); + mi_segment_schedule_purge(segment, mi_slice_start(slice), slice_count * MI_SEGMENT_SLICE_SIZE); } // and push it on the free page queue (if it was not a huge page) @@ -662,7 +681,6 @@ static void mi_segment_span_remove_from_queue(mi_slice_t* slice, mi_segments_tld static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_tld_t* tld) { mi_assert_internal(slice != NULL && slice->slice_count > 0 && slice->slice_offset == 0); mi_segment_t* const segment = _mi_ptr_segment(slice); - const bool is_abandoned = (segment->thread_id == 0); // mi_segment_is_abandoned(segment); // for huge pages, just mark as free but don't add to the queues if (segment->kind == MI_SEGMENT_HUGE) { @@ -675,6 +693,7 @@ static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_ } // otherwise coalesce the span and add to the free span queues + const bool is_abandoned = (segment->thread_id == 0); // mi_segment_is_abandoned(segment); size_t slice_count = slice->slice_count; mi_slice_t* next = slice + slice->slice_count; mi_assert_internal(next <= mi_segment_slices_end(segment)); @@ -691,6 +710,8 @@ static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_ // free previous slice -- remove it from free and merge mi_assert_internal(prev->slice_count > 0 && prev->slice_offset==0); slice_count += prev->slice_count; + slice->slice_count = 0; + slice->slice_offset = (uint32_t)((uint8_t*)slice - (uint8_t*)prev); // set the slice offset for `segment_force_abandon` (in case the previous free block is very large). if (!is_abandoned) { mi_segment_span_remove_from_queue(prev, tld); } slice = prev; } @@ -708,13 +729,13 @@ static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_ ----------------------------------------------------------- */ // Note: may still return NULL if committing the memory failed -static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_index, size_t slice_count, mi_segments_tld_t* tld) { +static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_index, size_t slice_count) { mi_assert_internal(slice_index < segment->slice_entries); mi_slice_t* const slice = &segment->slices[slice_index]; mi_assert_internal(slice->block_size==0 || slice->block_size==1); // commit before changing the slice data - if (!mi_segment_ensure_committed(segment, _mi_segment_page_start_from_slice(segment, slice, 0, NULL), slice_count * MI_SEGMENT_SLICE_SIZE, tld->stats)) { + if (!mi_segment_ensure_committed(segment, _mi_segment_page_start_from_slice(segment, slice, 0, NULL), slice_count * MI_SEGMENT_SLICE_SIZE)) { return NULL; // commit failed! } @@ -787,7 +808,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren mi_segment_slice_split(segment, slice, slice_count, tld); } mi_assert_internal(slice != NULL && slice->slice_count == slice_count && slice->block_size > 0); - mi_page_t* page = mi_segment_span_allocate(segment, mi_slice_index(slice), slice->slice_count, tld); + mi_page_t* page = mi_segment_span_allocate(segment, mi_slice_index(slice), slice->slice_count); if (page == NULL) { // commit failed; return NULL but first restore the slice mi_segment_span_free_coalesce(slice, tld); @@ -810,7 +831,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id, size_t* psegment_slices, size_t* pinfo_slices, - bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) + bool commit, mi_segments_tld_t* tld) { mi_memid_t memid; @@ -831,7 +852,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment } const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE; - mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, os_tld); + mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid); if (segment == NULL) { return NULL; // failed to allocate } @@ -847,8 +868,8 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment mi_assert_internal(commit_needed>0); mi_commit_mask_create(0, commit_needed, &commit_mask); mi_assert_internal(commit_needed*MI_COMMIT_SIZE >= (*pinfo_slices)*MI_SEGMENT_SLICE_SIZE); - if (!_mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, NULL, tld->stats)) { - _mi_arena_free(segment,segment_size,0,memid,tld->stats); + if (!_mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, NULL)) { + _mi_arena_free(segment,segment_size,0,memid); return NULL; } } @@ -858,6 +879,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment segment->allow_decommit = !memid.is_pinned; segment->allow_purge = segment->allow_decommit && (mi_option_get(mi_option_purge_delay) >= 0); segment->segment_size = segment_size; + segment->subproc = tld->subproc; segment->commit_mask = commit_mask; segment->purge_expire = 0; mi_commit_mask_create_empty(&segment->purge_mask); @@ -869,7 +891,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment // Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . -static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page) +static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_page_t** huge_page) { mi_assert_internal((required==0 && huge_page==NULL) || (required>0 && huge_page != NULL)); @@ -881,13 +903,13 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) const bool eager_delay = (// !_mi_os_has_overcommit() && // never delay on overcommit systems _mi_current_thread_count() > 1 && // do not delay for the first N threads - tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); + tld->peak_count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager || (required > 0); // Allocate the segment from the OS mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id, - &segment_slices, &info_slices, commit, tld, os_tld); + &segment_slices, &info_slices, commit, tld); if (segment == NULL) return NULL; // zero the segment info? -- not always needed as it may be zero initialized from the OS @@ -915,17 +937,17 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi if (MI_SECURE>0) { // in secure mode, we set up a protected page in between the segment info // and the page data, and at the end of the segment. - size_t os_pagesize = _mi_os_page_size(); + size_t os_pagesize = _mi_os_page_size(); _mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize); uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize; - mi_segment_ensure_committed(segment, end, os_pagesize, tld->stats); + mi_segment_ensure_committed(segment, end, os_pagesize); _mi_os_protect(end, os_pagesize); if (slice_entries == segment_slices) segment->slice_entries--; // don't use the last slice :-( guard_slices = 1; } // reserve first slices for segment info - mi_page_t* page0 = mi_segment_span_allocate(segment, 0, info_slices, tld); + mi_page_t* page0 = mi_segment_span_allocate(segment, 0, info_slices); mi_assert_internal(page0!=NULL); if (page0==NULL) return NULL; // cannot fail as we always commit in advance mi_assert_internal(segment->used == 1); segment->used = 0; // don't count our internal slices towards usage @@ -939,7 +961,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi mi_assert_internal(huge_page!=NULL); mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask)); mi_assert_internal(mi_commit_mask_is_full(&segment->commit_mask)); - *huge_page = mi_segment_span_allocate(segment, info_slices, segment_slices - info_slices - guard_slices, tld); + *huge_page = mi_segment_span_allocate(segment, info_slices, segment_slices - info_slices - guard_slices); mi_assert_internal(*huge_page != NULL); // cannot fail as we commit in advance } @@ -953,6 +975,9 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t mi_assert_internal(segment != NULL); mi_assert_internal(segment->next == NULL); mi_assert_internal(segment->used == 0); + + // in `mi_segment_force_abandon` we set this to true to ensure the segment's memory stays valid + if (segment->dont_free) return; // Remove the free pages mi_slice_t* slice = &segment->slices[0]; @@ -1003,7 +1028,7 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld if (segment->allow_decommit && mi_option_is_enabled(mi_option_deprecated_page_reset)) { size_t psize; uint8_t* start = _mi_segment_page_start(segment, page, &psize); - _mi_os_reset(start, psize, tld->stats); + _mi_os_reset(start, psize); } // zero the page data, but not the segment fields and heap tag @@ -1025,7 +1050,6 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) { mi_assert(page != NULL); - mi_segment_t* segment = _mi_page_segment(page); mi_assert_expensive(mi_segment_is_valid(segment,tld)); @@ -1043,7 +1067,7 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) } else { // perform delayed purges - mi_segment_try_purge(segment, false /* force? */, tld->stats); + mi_segment_try_purge(segment, false /* force? */); } } @@ -1061,16 +1085,11 @@ When a block is freed in an abandoned segment, the segment is reclaimed into that thread. Moreover, if threads are looking for a fresh segment, they -will first consider abondoned segments -- these can be found +will first consider abandoned segments -- these can be found by scanning the arena memory (segments outside arena memoryare only reclaimed by a free). ----------------------------------------------------------- */ -// legacy: Wait until there are no more pending reads on segments that used to be in the abandoned list -void _mi_abandoned_await_readers(void) { - // nothing needed -} - /* ----------------------------------------------------------- Abandon segment/page ----------------------------------------------------------- */ @@ -1098,7 +1117,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { // Only abandoned segments in arena memory can be reclaimed without a free // so if a segment is not from an arena we force purge here to be conservative. const bool force_purge = (segment->memid.memkind != MI_MEM_ARENA) || mi_option_is_enabled(mi_option_abandoned_page_purge); - mi_segment_try_purge(segment, force_purge, tld->stats); + mi_segment_try_purge(segment, force_purge); // all pages in the segment are abandoned; add it to the abandoned list _mi_stat_increase(&tld->stats->segments_abandoned, 1); @@ -1190,6 +1209,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, if (right_page_reclaimed != NULL) { *right_page_reclaimed = false; } // can be 0 still with abandoned_next, or already a thread id for segments outside an arena that are reclaimed on a free. mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0 || mi_atomic_load_relaxed(&segment->thread_id) == _mi_thread_id()); + mi_assert_internal(segment->subproc == heap->tld->segments.subproc); // only reclaim within the same subprocess mi_atomic_store_release(&segment->thread_id, _mi_thread_id()); segment->abandoned_visits = 0; segment->was_reclaimed = true; @@ -1213,12 +1233,13 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, mi_assert_internal(page->next == NULL && page->prev==NULL); _mi_stat_decrease(&tld->stats->pages_abandoned, 1); segment->abandoned--; - // set the heap again and allow heap thread delayed free again. + // get the target heap for this thread which has a matching heap tag (so we reclaim into a matching heap) mi_heap_t* target_heap = _mi_heap_by_tag(heap, page->heap_tag); // allow custom heaps to separate objects if (target_heap == NULL) { target_heap = heap; - _mi_error_message(EINVAL, "page with tag %u cannot be reclaimed by a heap with the same tag (using %u instead)\n", page->heap_tag, heap->tag ); + _mi_error_message(EFAULT, "page with tag %u cannot be reclaimed by a heap with the same tag (using heap tag %u instead)\n", page->heap_tag, heap->tag ); } + // associate the heap with this page, and allow heap thread delayed free again. mi_page_set_heap(page, target_heap); _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) _mi_page_free_collect(page, false); // ensure used count is up to date @@ -1254,12 +1275,21 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, } } + // attempt to reclaim a particular segment (called from multi threaded free `alloc.c:mi_free_block_mt`) bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { if (mi_atomic_load_relaxed(&segment->thread_id) != 0) return false; // it is not abandoned - // don't reclaim more from a free than half the current segments + if (segment->subproc != heap->tld->segments.subproc) return false; // only reclaim within the same subprocess + if (!_mi_heap_memid_is_suitable(heap,segment->memid)) return false; // don't reclaim between exclusive and non-exclusive arena's + const long target = _mi_option_get_fast(mi_option_target_segments_per_thread); + if (target > 0 && (size_t)target <= heap->tld->segments.count) return false; // don't reclaim if going above the target count + + // don't reclaim more from a `free` call than half the current segments // this is to prevent a pure free-ing thread to start owning too many segments - if (heap->tld->segments.reclaim_count * 2 > heap->tld->segments.count) return false; + // (but not for out-of-arena segments as that is the main way to be reclaimed for those) + if (segment->memid.memkind == MI_MEM_ARENA && heap->tld->segments.reclaim_count * 2 > heap->tld->segments.count) { + return false; + } if (_mi_arena_segment_clear_abandoned(segment)) { // atomically unabandon mi_segment_t* res = mi_segment_reclaim(segment, heap, 0, NULL, &heap->tld->segments); mi_assert_internal(res == segment); @@ -1270,17 +1300,26 @@ bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { mi_segment_t* segment; - mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); + mi_arena_field_cursor_t current; + _mi_arena_field_cursor_init(heap, tld->subproc, true /* visit all, blocking */, ¤t); while ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) { mi_segment_reclaim(segment, heap, 0, NULL, tld); } + _mi_arena_field_cursor_done(¤t); +} + + +static bool segment_count_is_within_target(mi_segments_tld_t* tld, size_t* ptarget) { + const size_t target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread, 0, 1024); + if (ptarget != NULL) { *ptarget = target; } + return (target == 0 || tld->count < target); } -static long mi_segment_get_reclaim_tries(void) { +static long mi_segment_get_reclaim_tries(mi_segments_tld_t* tld) { // limit the tries to 10% (default) of the abandoned segments with at least 8 and at most 1024 tries. const size_t perc = (size_t)mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 100); if (perc <= 0) return 0; - const size_t total_count = _mi_arena_segment_abandoned_count(); + const size_t total_count = mi_atomic_load_relaxed(&tld->subproc->abandoned_count); if (total_count == 0) return 0; const size_t relative_count = (total_count > 10000 ? (total_count / 100) * perc : (total_count * perc) / 100); // avoid overflow long max_tries = (long)(relative_count <= 1 ? 1 : (relative_count > 1024 ? 1024 : relative_count)); @@ -1291,15 +1330,18 @@ static long mi_segment_get_reclaim_tries(void) { static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slices, size_t block_size, bool* reclaimed, mi_segments_tld_t* tld) { *reclaimed = false; - long max_tries = mi_segment_get_reclaim_tries(); + long max_tries = mi_segment_get_reclaim_tries(tld); if (max_tries <= 0) return NULL; - mi_segment_t* segment; - mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); - while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) + mi_segment_t* result = NULL; + mi_segment_t* segment = NULL; + mi_arena_field_cursor_t current; + _mi_arena_field_cursor_init(heap, tld->subproc, false /* non-blocking */, ¤t); + while (segment_count_is_within_target(tld,NULL) && (max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) { + mi_assert(segment->subproc == heap->tld->segments.subproc); // cursor only visits segments in our sub-process segment->abandoned_visits++; - // todo: should we respect numa affinity for abondoned reclaim? perhaps only for the first visit? + // todo: should we respect numa affinity for abandoned reclaim? perhaps only for the first visit? // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments and use many tries // Perhaps we can skip non-suitable ones in a better way? bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid); @@ -1316,27 +1358,29 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice // found a large enough free span, or a page of the right block_size with free space // we return the result of reclaim (which is usually `segment`) as it might free // the segment due to concurrent frees (in which case `NULL` is returned). - return mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); + result = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); + break; } else if (segment->abandoned_visits > 3 && is_suitable) { - // always reclaim on 3rd visit to limit the abandoned queue length. + // always reclaim on 3rd visit to limit the abandoned segment count. mi_segment_reclaim(segment, heap, 0, NULL, tld); } else { // otherwise, push on the visited list so it gets not looked at too quickly again - mi_segment_try_purge(segment, false /* true force? */, tld->stats); // force purge if needed as we may not visit soon again + mi_segment_try_purge(segment, false /* true force? */); // force purge if needed as we may not visit soon again _mi_arena_segment_mark_abandoned(segment); } } - return NULL; + _mi_arena_field_cursor_done(¤t); + return result; } - +// collect abandoned segments void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) { mi_segment_t* segment; - mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); - long max_tries = (force ? (long)_mi_arena_segment_abandoned_count() : 1024); // limit latency + mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, tld->subproc, force /* blocking? */, ¤t); + long max_tries = (force ? (long)mi_atomic_load_relaxed(&tld->subproc->abandoned_count) : 1024); // limit latency while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) { mi_segment_check_free(segment,0,0,tld); // try to free up pages (due to concurrent frees) if (segment->used == 0) { @@ -1348,20 +1392,121 @@ void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) else { // otherwise, purge if needed and push on the visited list // note: forced purge can be expensive if many threads are destroyed/created as in mstress. - mi_segment_try_purge(segment, force, tld->stats); + mi_segment_try_purge(segment, force); _mi_arena_segment_mark_abandoned(segment); } } + _mi_arena_field_cursor_done(¤t); +} + +/* ----------------------------------------------------------- + Force abandon a segment that is in use by our thread +----------------------------------------------------------- */ + +// force abandon a segment +static void mi_segment_force_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) +{ + mi_assert_internal(!mi_segment_is_abandoned(segment)); + mi_assert_internal(!segment->dont_free); + + // ensure the segment does not get free'd underneath us (so we can check if a page has been freed in `mi_page_force_abandon`) + segment->dont_free = true; + + // for all slices + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + mi_assert_internal(slice->slice_count > 0); + mi_assert_internal(slice->slice_offset == 0); + if (mi_slice_is_used(slice)) { + // ensure used count is up to date and collect potential concurrent frees + mi_page_t* const page = mi_slice_to_page(slice); + _mi_page_free_collect(page, false); + { + // abandon the page if it is still in-use (this will free it if possible as well) + mi_assert_internal(segment->used > 0); + if (segment->used == segment->abandoned+1) { + // the last page.. abandon and return as the segment will be abandoned after this + // and we should no longer access it. + segment->dont_free = false; + _mi_page_force_abandon(page); + return; + } + else { + // abandon and continue + _mi_page_force_abandon(page); + // it might be freed, reset the slice (note: relies on coalesce setting the slice_offset) + slice = mi_slice_first(slice); + } + } + } + slice = slice + slice->slice_count; + } + segment->dont_free = false; + mi_assert(segment->used == segment->abandoned); + mi_assert(segment->used == 0); + if (segment->used == 0) { // paranoia + // all free now + mi_segment_free(segment, false, tld); + } + else { + // perform delayed purges + mi_segment_try_purge(segment, false /* force? */); + } +} + + +// try abandon segments. +// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use. +static void mi_segments_try_abandon_to_target(mi_heap_t* heap, size_t target, mi_segments_tld_t* tld) { + if (target <= 1) return; + const size_t min_target = (target > 4 ? (target*3)/4 : target); // 75% + // todo: we should maintain a list of segments per thread; for now, only consider segments from the heap full pages + for (int i = 0; i < 64 && tld->count >= min_target; i++) { + mi_page_t* page = heap->pages[MI_BIN_FULL].first; + while (page != NULL && mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX) { + page = page->next; + } + if (page==NULL) { + break; + } + mi_segment_t* segment = _mi_page_segment(page); + mi_segment_force_abandon(segment, tld); + mi_assert_internal(page != heap->pages[MI_BIN_FULL].first); // as it is just abandoned + } +} + +// try abandon segments. +// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use. +static void mi_segments_try_abandon(mi_heap_t* heap, mi_segments_tld_t* tld) { + // we call this when we are about to add a fresh segment so we should be under our target segment count. + size_t target = 0; + if (segment_count_is_within_target(tld, &target)) return; + mi_segments_try_abandon_to_target(heap, target, tld); +} + +void mi_collect_reduce(size_t target_size) mi_attr_noexcept { + mi_collect(true); + mi_heap_t* heap = mi_heap_get_default(); + mi_segments_tld_t* tld = &heap->tld->segments; + size_t target = target_size / MI_SEGMENT_SIZE; + if (target == 0) { + target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread, 1, 1024); + } + mi_segments_try_abandon_to_target(heap, target, tld); } /* ----------------------------------------------------------- Reclaim or allocate ----------------------------------------------------------- */ -static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_slices, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_slices, size_t block_size, mi_segments_tld_t* tld) { mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX); + // try to abandon some segments to increase reuse between threads + mi_segments_try_abandon(heap,tld); + // 1. try to reclaim an abandoned segment bool reclaimed; mi_segment_t* segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld); @@ -1375,7 +1520,7 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_ return segment; } // 2. otherwise allocate a fresh segment - return mi_segment_alloc(0, 0, heap->arena_id, tld, os_tld, NULL); + return mi_segment_alloc(0, 0, heap->arena_id, tld, NULL); } @@ -1383,7 +1528,7 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_ Page allocation ----------------------------------------------------------- */ -static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_kind, size_t required, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_kind, size_t required, size_t block_size, mi_segments_tld_t* tld) { mi_assert_internal(required <= MI_LARGE_OBJ_SIZE_MAX && page_kind <= MI_PAGE_LARGE); @@ -1394,18 +1539,18 @@ static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_ki mi_page_t* page = mi_segments_page_find_and_allocate(slices_needed, heap->arena_id, tld); //(required <= MI_SMALL_SIZE_MAX ? 0 : slices_needed), tld); if (page==NULL) { // no free page, allocate a new segment and try again - if (mi_segment_reclaim_or_alloc(heap, slices_needed, block_size, tld, os_tld) == NULL) { + if (mi_segment_reclaim_or_alloc(heap, slices_needed, block_size, tld) == NULL) { // OOM or reclaimed a good page in the heap return NULL; } else { // otherwise try again - return mi_segments_page_alloc(heap, page_kind, required, block_size, tld, os_tld); + return mi_segments_page_alloc(heap, page_kind, required, block_size, tld); } } mi_assert_internal(page != NULL && page->slice_count*MI_SEGMENT_SLICE_SIZE == page_size); mi_assert_internal(_mi_ptr_segment(page)->thread_id == _mi_thread_id()); - mi_segment_try_purge(_mi_ptr_segment(page), false, tld->stats); + mi_segment_try_purge(_mi_ptr_segment(page), false); return page; } @@ -1415,10 +1560,10 @@ static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_ki Huge page allocation ----------------------------------------------------------- */ -static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) { mi_page_t* page = NULL; - mi_segment_t* segment = mi_segment_alloc(size,page_alignment,req_arena_id,tld,os_tld,&page); + mi_segment_t* segment = mi_segment_alloc(size,page_alignment,req_arena_id,tld,&page); if (segment == NULL || page==NULL) return NULL; mi_assert_internal(segment->used==1); mi_assert_internal(mi_page_block_size(page) >= size); @@ -1440,7 +1585,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_assert_internal(psize - (aligned_p - start) >= size); uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list ptrdiff_t decommit_size = aligned_p - decommit_start; - _mi_os_reset(decommit_start, decommit_size, &_mi_stats_main); // note: cannot use segment_decommit on huge segments + _mi_os_reset(decommit_start, decommit_size); // note: cannot use segment_decommit on huge segments } return page; @@ -1487,7 +1632,7 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc if (csize > sizeof(mi_block_t)) { csize = csize - sizeof(mi_block_t); uint8_t* p = (uint8_t*)block + sizeof(mi_block_t); - _mi_os_reset(p, csize, &_mi_stats_main); // note: cannot use segment_decommit on huge segments + _mi_os_reset(p, csize); // note: cannot use segment_decommit on huge segments } } } @@ -1496,29 +1641,60 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc /* ----------------------------------------------------------- Page allocation and free ----------------------------------------------------------- */ -mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { +mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld) { mi_page_t* page; if mi_unlikely(page_alignment > MI_BLOCK_ALIGNMENT_MAX) { mi_assert_internal(_mi_is_power_of_two(page_alignment)); mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE); if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; } - page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld,os_tld); + page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld); } else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) { - page = mi_segments_page_alloc(heap,MI_PAGE_SMALL,block_size,block_size,tld,os_tld); + page = mi_segments_page_alloc(heap,MI_PAGE_SMALL,block_size,block_size,tld); } else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) { - page = mi_segments_page_alloc(heap,MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,block_size,tld, os_tld); + page = mi_segments_page_alloc(heap,MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,block_size,tld); } else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) { - page = mi_segments_page_alloc(heap,MI_PAGE_LARGE,block_size,block_size,tld, os_tld); + page = mi_segments_page_alloc(heap,MI_PAGE_LARGE,block_size,block_size,tld); } else { - page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld,os_tld); + page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld); } mi_assert_internal(page == NULL || _mi_heap_memid_is_suitable(heap, _mi_page_segment(page)->memid)); mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld)); + mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc); return page; } +/* ----------------------------------------------------------- + Visit blocks in a segment (only used for abandoned segments) +----------------------------------------------------------- */ + +static bool mi_segment_visit_page(mi_page_t* page, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + mi_heap_area_t area; + _mi_heap_area_init(&area, page); + if (!visitor(NULL, &area, NULL, area.block_size, arg)) return false; + if (visit_blocks) { + return _mi_heap_area_visit_blocks(&area, page, visitor, arg); + } + else { + return true; + } +} + +bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { + const mi_slice_t* end; + mi_slice_t* slice = mi_slices_start_iterate(segment, &end); + while (slice < end) { + if (mi_slice_is_used(slice)) { + mi_page_t* const page = mi_slice_to_page(slice); + if (heap_tag < 0 || (int)page->heap_tag == heap_tag) { + if (!mi_segment_visit_page(page, visit_blocks, visitor, arg)) return false; + } + } + slice = slice + slice->slice_count; + } + return true; +} diff --git a/third-party/mimalloc/src/static.c b/third-party/mimalloc/src/static.c index bf025eb794..9e06ce05aa 100644 --- a/third-party/mimalloc/src/static.c +++ b/third-party/mimalloc/src/static.c @@ -31,7 +31,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "options.c" #include "os.c" #include "page.c" // includes page-queue.c -#include "random.c" +#include "random.c" #include "segment.c" #include "segment-map.c" #include "stats.c" diff --git a/third-party/mimalloc/src/stats.c b/third-party/mimalloc/src/stats.c index a936402744..f82055938c 100644 --- a/third-party/mimalloc/src/stats.c +++ b/third-party/mimalloc/src/stats.c @@ -26,7 +26,7 @@ static bool mi_is_in_main(void* stat) { static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; - if (mi_is_in_main(stat)) + if mi_unlikely(mi_is_in_main(stat)) { // add atomically (for abandoned pages) int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount); @@ -51,6 +51,27 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { } } +// Adjust stats to compensate; for example before committing a range, +// first adjust downwards with parts that were already committed so +// we avoid double counting. +static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { + if (amount == 0) return; + if mi_unlikely(mi_is_in_main(stat)) + { + // adjust atomically + mi_atomic_addi64_relaxed(&stat->current, amount); + mi_atomic_addi64_relaxed(&stat->allocated, amount); + mi_atomic_addi64_relaxed(&stat->freed, amount); + } + else { + // don't affect the peak + stat->current += amount; + // add to both + stat->allocated += amount; + stat->freed += amount; + } +} + void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { if (mi_is_in_main(stat)) { mi_atomic_addi64_relaxed( &stat->count, 1 ); @@ -70,6 +91,14 @@ void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { mi_stat_update(stat, -((int64_t)amount)); } +void _mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) { + mi_stat_adjust(stat, (int64_t)amount); +} + +void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { + mi_stat_adjust(stat, -((int64_t)amount)); +} + // must be thread safe as it is called from stats_merge static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) { if (stat==src) return; @@ -119,6 +148,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); mi_stat_counter_add(&stats->large_count, &src->large_count, 1); + mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { @@ -345,6 +375,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); + mi_stat_counter_print(&stats->guarded_alloc_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); diff --git a/third-party/mimalloc/test/main-override-static.c b/third-party/mimalloc/test/main-override-static.c index e71be29e95..1fee245221 100644 --- a/third-party/mimalloc/test/main-override-static.c +++ b/third-party/mimalloc/test/main-override-static.c @@ -1,3 +1,6 @@ +#if _WIN32 +#include +#endif #include #include #include @@ -12,6 +15,7 @@ static void double_free1(); static void double_free2(); static void corrupt_free(); static void block_overflow1(); +static void block_overflow2(); static void invalid_free(); static void test_aslr(void); static void test_process_info(void); @@ -21,15 +25,22 @@ static void alloc_huge(void); static void test_heap_walk(void); static void test_heap_arena(void); static void test_align(void); +static void test_canary_leak(void); +static void test_manage_os_memory(void); +// static void test_large_pages(void); int main() { mi_version(); mi_stats_reset(); + test_manage_os_memory(); + // test_large_pages(); // detect double frees and heap corruption // double_free1(); // double_free2(); // corrupt_free(); // block_overflow1(); + // block_overflow2(); + test_canary_leak(); // test_aslr(); // invalid_free(); // test_reserved(); @@ -66,7 +77,7 @@ int main() { //mi_stats_print(NULL); // test_process_info(); - + return 0; } @@ -88,6 +99,12 @@ static void block_overflow1() { free(p); } +static void block_overflow2() { + uint8_t* p = (uint8_t*)mi_malloc(16); + p[17] = 0; + free(p); +} + // The double free samples come ArcHeap [1] by Insu Yun (issue #161) // [1]: https://arxiv.org/pdf/1903.00503.pdf @@ -242,6 +259,78 @@ static void test_heap_arena(void) { } } +static void test_canary_leak(void) { + char* p = mi_mallocn_tp(char,23); + for(int i = 0; i < 23; i++) { + p[i] = '0'+i; + } + puts(p); + free(p); +} + +#if _WIN32 +static void test_manage_os_memory(void) { + size_t size = 256 * 1024 * 1024; + void* ptr = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + mi_arena_id_t arena_id; + mi_manage_os_memory_ex(ptr, size, true /* committed */, true /* pinned */, false /* is zero */, -1 /* numa node */, true /* exclusive */, &arena_id); + mi_heap_t* cuda_heap = mi_heap_new_in_arena(arena_id); // you can do this in any thread + + // now allocate only in the cuda arena + void* p1 = mi_heap_malloc(cuda_heap, 8); + int* p2 = mi_heap_malloc_tp(cuda_heap, int); + *p2 = 42; + + // and maybe set the cuda heap as the default heap? (but careful as now `malloc` will allocate in the cuda heap as well) + { + mi_heap_t* prev_default_heap = mi_heap_set_default(cuda_heap); + void* p3 = mi_malloc(8); // allocate in the cuda heap + mi_free(p3); + } + mi_free(p1); + mi_free(p2); +} +#else +static void test_manage_os_memory(void) { + // empty +} +#endif + +// Experiment with huge OS pages +#if 0 + +#include +#include +#include +#include + +static void test_large_pages(void) { + mi_memid_t memid; + + #if 0 + size_t pages_reserved; + size_t page_size; + uint8_t* p = (uint8_t*)_mi_os_alloc_huge_os_pages(1, -1, 30000, &pages_reserved, &page_size, &memid); + const size_t req_size = pages_reserved * page_size; + #else + const size_t req_size = 64*MI_MiB; + uint8_t* p = (uint8_t*)_mi_os_alloc(req_size,&memid,NULL); + #endif + + p[0] = 1; + + //_mi_os_protect(p, _mi_os_page_size()); + //_mi_os_unprotect(p, _mi_os_page_size()); + //_mi_os_decommit(p, _mi_os_page_size(), NULL); + if (madvise(p, req_size, MADV_HUGEPAGE) == 0) { + printf("advised huge pages\n"); + _mi_os_decommit(p, _mi_os_page_size(), NULL); + }; + _mi_os_free(p, req_size, memid, NULL); +} + +#endif + // ---------------------------- // bin size experiments // ------------------------------ @@ -260,11 +349,11 @@ static void test_heap_arena(void) { static inline uint8_t mi_bsr32(uint32_t x); #if defined(_MSC_VER) -#include +//#include #include static inline uint8_t mi_bsr32(uint32_t x) { uint32_t idx; - _BitScanReverse((DWORD*)&idx, x); + _BitScanReverse(&idx, x); return idx; } #elif defined(__GNUC__) || defined(__clang__) @@ -288,7 +377,7 @@ static inline uint8_t mi_bsr32(uint32_t x) { } #endif -/* + // Bit scan reverse: return the index of the highest bit. uint8_t _mi_bsr(uintptr_t x) { if (x == 0) return 0; @@ -301,7 +390,7 @@ uint8_t _mi_bsr(uintptr_t x) { # error "define bsr for non-32 or 64-bit platforms" #endif } -*/ + static inline size_t _mi_wsize_from_size(size_t size) { @@ -378,11 +467,20 @@ static inline uint8_t _mi_bin4(size_t size) { return bin; } -static size_t _mi_binx4(size_t bsize) { - if (bsize==0) return 0; - uint8_t b = mi_bsr32((uint32_t)bsize); - if (b <= 1) return bsize; - size_t bin = ((b << 1) | (bsize >> (b - 1))&0x01); +static size_t _mi_binx4(size_t wsize) { + size_t bin; + if (wsize <= 1) { + bin = 1; + } + else if (wsize <= 8) { + // bin = (wsize+1)&~1; // round to double word sizes + bin = (uint8_t)wsize; + } + else { + uint8_t b = mi_bsr32((uint32_t)wsize); + if (b <= 1) return wsize; + bin = ((b << 1) | (wsize >> (b - 1))&0x01) + 3; + } return bin; } @@ -394,22 +492,40 @@ static size_t _mi_binx8(size_t bsize) { return bin; } + +static inline size_t mi_bin(size_t wsize) { + uint8_t bin; + if (wsize <= 1) { + bin = 1; + } + else if (wsize <= 8) { + // bin = (wsize+1)&~1; // round to double word sizes + bin = (uint8_t)wsize; + } + else { + wsize--; + // find the highest bit + uint8_t b = (uint8_t)mi_bsr32((uint32_t)wsize); // note: wsize != 0 + // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). + // - adjust with 3 because we use do not round the first 8 sizes + // which each get an exact bin + bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; + } + return bin; +} + + static void mi_bins(void) { //printf(" QNULL(1), /* 0 */ \\\n "); size_t last_bin = 0; - size_t min_bsize = 0; - size_t last_bsize = 0; - for (size_t bsize = 1; bsize < 2*1024; bsize++) { - size_t size = bsize * 64 * 1024; - size_t bin = _mi_binx8(bsize); + for (size_t wsize = 1; wsize <= (4*1024*1024) / 8 + 1024; wsize++) { + size_t bin = mi_bin(wsize); if (bin != last_bin) { - printf("min bsize: %6zd, max bsize: %6zd, bin: %6zd\n", min_bsize, last_bsize, last_bin); - //printf("QNULL(%6zd), ", wsize); - //if (last_bin%8 == 0) printf("/* %i */ \\\n ", last_bin); + //printf("min bsize: %6zd, max bsize: %6zd, bin: %6zd\n", min_wsize, last_wsize, last_bin); + printf("QNULL(%6zd), ", wsize-1); + if (last_bin%8 == 0) printf("/* %zu */ \\\n ", last_bin); last_bin = bin; - min_bsize = bsize; } - last_bsize = bsize; } } #endif diff --git a/third-party/mimalloc/test/main-override.cpp b/third-party/mimalloc/test/main-override.cpp index 582f24ee92..0fbb58e877 100644 --- a/third-party/mimalloc/test/main-override.cpp +++ b/third-party/mimalloc/test/main-override.cpp @@ -11,7 +11,7 @@ #include #include -#include +//#include #include #ifdef _WIN32 @@ -19,7 +19,7 @@ #endif #ifdef _WIN32 -#include +#include static void msleep(unsigned long msecs) { Sleep(msecs); } #else #include @@ -35,19 +35,24 @@ static void test_mt_shutdown(); static void large_alloc(void); // issue #363 static void fail_aslr(); // issue #372 static void tsan_numa_test(); // issue #414 -static void strdup_test(); // issue #445 +static void strdup_test(); // issue #445 static void bench_alloc_large(void); // issue #xxx //static void test_large_migrate(void); // issue #691 static void heap_thread_free_huge(); static void test_std_string(); // issue #697 - +static void test_thread_local(); // issue #944 +// static void test_mixed0(); // issue #942 +static void test_mixed1(); // issue #942 static void test_stl_allocators(); int main() { - // mi_stats_reset(); // ignore earlier allocations + mi_stats_reset(); // ignore earlier allocations + various_tests(); + test_mixed1(); - // test_std_string(); + //test_std_string(); + //test_thread_local(); // heap_thread_free_huge(); /* heap_thread_free_huge(); @@ -63,10 +68,9 @@ int main() { // test_stl_allocators(); // test_mt_shutdown(); // test_large_migrate(); - + //fail_aslr(); - // bench_alloc_large(); - // mi_stats_print(NULL); + mi_stats_print(NULL); return 0; } @@ -109,6 +113,9 @@ static void various_tests() { t = new (tbuf) Test(42); t->~Test(); delete[] tbuf; + + const char* ptr = ::_Getdays(); // test _base overrid + free((void*)ptr); } class Static { @@ -185,6 +192,53 @@ static void test_stl_allocators() { #endif } +#if 0 +#include +#include +#include +#include +#include +#include + +static void test_mixed0() { + std::vector> numbers(1024 * 1024 * 100); + std::vector threads(1); + + std::atomic index{}; + + auto start = std::chrono::system_clock::now(); + + for (auto& thread : threads) { + thread = std::thread{[&index, &numbers]() { + while (true) { + auto i = index.fetch_add(1, std::memory_order_relaxed); + if (i >= numbers.size()) return; + + numbers[i] = std::make_unique(i); + } + }}; + } + + for (auto& thread : threads) thread.join(); + + auto end = std::chrono::system_clock::now(); + + auto duration = + std::chrono::duration_cast(end - start); + std::cout << "Running on " << threads.size() << " threads took " << duration + << std::endl; +} +#endif + +void asd() { + void* p = malloc(128); + free(p); +} +static void test_mixed1() { + std::thread thread(asd); + thread.join(); +} + #if 0 // issue #691 static char* cptr; @@ -398,3 +452,30 @@ static void bench_alloc_large(void) { std::cout << "Avg " << us_per_allocation << " us per allocation" << std::endl; } + +class MTest +{ + char *data; +public: + MTest() { data = (char*)malloc(1024); } + ~MTest() { free(data); }; +}; + +thread_local MTest tlVariable; + +void threadFun( int i ) +{ + printf( "Thread %d\n", i ); + std::this_thread::sleep_for( std::chrono::milliseconds(100) ); +} + +void test_thread_local() +{ + for( int i=1; i < 100; ++i ) + { + std::thread t( threadFun, i ); + t.join(); + mi_stats_print(NULL); + } + return; +} diff --git a/third-party/mimalloc/test/test-api-fill.c b/third-party/mimalloc/test/test-api-fill.c index 3fca3b9d43..eebbd394ef 100644 --- a/third-party/mimalloc/test/test-api-fill.c +++ b/third-party/mimalloc/test/test-api-fill.c @@ -271,7 +271,7 @@ int main(void) { mi_free(p); }; - #if !(MI_TRACK_VALGRIND || MI_TRACK_ASAN) + #if !(MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_GUARDED) CHECK_BODY("fill-freed-small") { size_t malloc_size = MI_SMALL_SIZE_MAX / 2; uint8_t* p = (uint8_t*)mi_malloc(malloc_size); diff --git a/third-party/mimalloc/test/test-api.c b/third-party/mimalloc/test/test-api.c index 76101980be..154845447f 100644 --- a/third-party/mimalloc/test/test-api.c +++ b/third-party/mimalloc/test/test-api.c @@ -65,6 +65,15 @@ bool mem_is_zero(uint8_t* p, size_t size) { int main(void) { mi_option_disable(mi_option_verbose); + CHECK_BODY("malloc-aligned9a") { // test large alignments + void* p = mi_zalloc_aligned(1024 * 1024, 2); + mi_free(p); + p = mi_zalloc_aligned(1024 * 1024, 2); + mi_free(p); + result = true; + }; + + // --------------------------------------------------- // Malloc // --------------------------------------------------- @@ -157,6 +166,7 @@ int main(void) { printf("malloc_aligned5: usable size: %zi\n", usable); mi_free(p); }; + /* CHECK_BODY("malloc-aligned6") { bool ok = true; for (size_t align = 1; align <= MI_BLOCK_ALIGNMENT_MAX && ok; align *= 2) { @@ -174,6 +184,7 @@ int main(void) { } result = ok; }; + */ CHECK_BODY("malloc-aligned7") { void* p = mi_malloc_aligned(1024,MI_BLOCK_ALIGNMENT_MAX); mi_free(p); @@ -189,7 +200,7 @@ int main(void) { } result = ok; }; - CHECK_BODY("malloc-aligned9") { + CHECK_BODY("malloc-aligned9") { // test large alignments bool ok = true; void* p[8]; size_t sizes[8] = { 8, 512, 1024 * 1024, MI_BLOCK_ALIGNMENT_MAX, MI_BLOCK_ALIGNMENT_MAX + 1, 2 * MI_BLOCK_ALIGNMENT_MAX, 8 * MI_BLOCK_ALIGNMENT_MAX, 0 }; diff --git a/third-party/mimalloc/test/test-stress.c b/third-party/mimalloc/test/test-stress.c index 15d0e25bf0..c0d9761ac5 100644 --- a/third-party/mimalloc/test/test-stress.c +++ b/third-party/mimalloc/test/test-stress.c @@ -22,29 +22,39 @@ terms of the MIT license. #include #include +// #define MI_GUARDED +// #define USE_STD_MALLOC + // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // // argument defaults -static int THREADS = 32; // more repeatable if THREADS <= #processors -static int SCALE = 25; // scaling factor - -#if defined(MI_TSAN) -static int ITER = 10; // N full iterations destructing and re-creating all threads (on tsan reduce for azure pipeline limits) +#if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits +static int THREADS = 8; +static int SCALE = 25; +static int ITER = 400; +#elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits +static int THREADS = 8; +static int SCALE = 25; +static int ITER = 20; +#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +static int THREADS = 8; +static int SCALE = 10; +static int ITER = 10; #else +static int THREADS = 32; // more repeatable if THREADS <= #processors +static int SCALE = 50; // scaling factor static int ITER = 50; // N full iterations destructing and re-creating all threads #endif -// static int THREADS = 8; // more repeatable if THREADS <= #processors -// static int SCALE = 100; // scaling factor + #define STRESS // undefine for leak test -static bool allow_large_objects = true; // allow very large objects? (set to `true` if SCALE>100) +static bool allow_large_objects = false; // allow very large objects? (set to `true` if SCALE>100) static size_t use_one_size = 0; // use single object size of `N * sizeof(uintptr_t)`? static bool main_participates = false; // main thread participates as a worker too -// #define USE_STD_MALLOC #ifdef USE_STD_MALLOC #define custom_calloc(n,s) calloc(n,s) #define custom_realloc(p,s) realloc(p,s) @@ -54,6 +64,10 @@ static bool main_participates = false; // main thread participates as a #define custom_calloc(n,s) mi_calloc(n,s) #define custom_realloc(p,s) mi_realloc(p,s) #define custom_free(p) mi_free(p) + +#ifndef NDEBUG +#define HEAP_WALK // walk the heap objects? +#endif #endif // transfer pointer between threads @@ -129,6 +143,16 @@ static void free_items(void* p) { custom_free(p); } +#ifdef HEAP_WALK +static bool visit_blocks(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { + (void)(heap); (void)(area); + size_t* total = (size_t*)arg; + if (block != NULL) { + *total += block_size; + } + return true; +} +#endif static void stress(intptr_t tid) { //bench_start_thread(); @@ -173,6 +197,13 @@ static void stress(intptr_t tid) { data[data_idx] = q; } } + + #ifdef HEAP_WALK + // walk the heap + size_t total = 0; + mi_heap_visit_blocks(mi_heap_get_default(), true, visit_blocks, &total); + #endif + // free everything that is left for (size_t i = 0; i < retain_top; i++) { free_items(retained[i]); @@ -190,7 +221,15 @@ static void run_os_threads(size_t nthreads, void (*entry)(intptr_t tid)); static void test_stress(void) { uintptr_t r = rand(); for (int n = 0; n < ITER; n++) { - run_os_threads(THREADS, &stress); + run_os_threads(THREADS, &stress); + #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) + // switch between arena and OS allocation for testing + // mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1); + #endif + #ifdef HEAP_WALK + size_t total = 0; + mi_abandoned_visit_blocks(mi_subproc_main(), -1, true, visit_blocks, &total); + #endif for (int i = 0; i < TRANSFERS; i++) { if (chance(50, &r) || n + 1 == ITER) { // free all on last run, otherwise free half of the transfers void* p = atomic_exchange_ptr(&transfer[i], NULL); @@ -199,8 +238,8 @@ static void test_stress(void) { } #ifndef NDEBUG //mi_collect(false); - //mi_debug_show_arenas(); - #endif + //mi_debug_show_arenas(true); + #endif #if !defined(NDEBUG) || defined(MI_TSAN) if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); } #endif @@ -230,9 +269,15 @@ static void test_leak(void) { #endif int main(int argc, char** argv) { + #ifdef HEAP_WALK + mi_option_enable(mi_option_visit_abandoned); + #endif + #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) + mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */); + #endif #ifndef USE_STD_MALLOC mi_stats_reset(); - #endif + #endif // > mimalloc-test-stress [THREADS] [SCALE] [ITER] if (argc >= 2) { @@ -277,11 +322,11 @@ int main(int argc, char** argv) { #ifndef USE_STD_MALLOC #ifndef NDEBUG - // mi_collect(true); - mi_debug_show_arenas(true,true,true); - #endif - mi_stats_print(NULL); + mi_debug_show_arenas(true); + mi_collect(true); + #endif #endif + mi_stats_print(NULL); //bench_end_program(); return 0; } @@ -291,7 +336,7 @@ static void (*thread_entry_fun)(intptr_t) = &stress; #ifdef _WIN32 -#include +#include static DWORD WINAPI thread_entry(LPVOID param) { thread_entry_fun((intptr_t)param);