diff --git a/CMakeLists.txt b/CMakeLists.txt index 24a6d53c3..16c0a3531 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,19 @@ if(CONST_QUALIFIED_MALLOC_USABLE_SIZE) endif() +# Build a redirection layer for all sizes that are a multiple of +# 16bytes up to 1024. +add_executable(generate src/redirect/generate.cc) +target_link_libraries(generate snmalloc_lib) +add_custom_target(generated ALL + COMMAND generate ${CMAKE_CURRENT_BINARY_DIR}/generated.cc + BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/generated.cc + ) +add_library(redirect_small STATIC src/redirect/redirect.cc) +target_link_libraries(redirect_small snmalloc_lib) +target_include_directories(redirect_small PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +add_dependencies(redirect_small generated) + # To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY # in containing Cmake file. if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) diff --git a/src/mem/alloc.h b/src/mem/alloc.h index 6ec2f8bae..666eb4e6c 100644 --- a/src/mem/alloc.h +++ b/src/mem/alloc.h @@ -1040,7 +1040,8 @@ namespace snmalloc allow_reserve == NoReserve ? "noreserve" : "reserve")); SNMALLOC_ASSUME(size <= SLAB_SIZE); - sizeclass_t sizeclass = size_to_sizeclass(size); + SNMALLOC_ASSUME(size > 0); + sizeclass_t sizeclass = size_to_sizeclass(size); return small_alloc_inner(sizeclass, size); } @@ -1066,6 +1067,13 @@ namespace snmalloc return p; } + return small_alloc_inner_slow(sizeclass, size); + } + + template + SNMALLOC_SLOW_PATH void* + small_alloc_inner_slow(sizeclass_t sizeclass, size_t size) + { if (likely(!has_messages())) return small_alloc_next_free_list( sizeclass, size); @@ -1228,6 +1236,20 @@ namespace snmalloc small_dealloc_offseted_inner(super, p, sizeclass); } + static SNMALLOC_FAST_PATH bool small_local_dealloc(void* p) + { + auto super = Superslab::get(p); + Slab* slab = Metaslab::get_slab(p); + return (likely(slab->dealloc_fast(super, p))); + } + + SNMALLOC_FAST_PATH void small_local_dealloc_slow(void* p) + { + auto super = Superslab::get(p); + Slab* slab = Metaslab::get_slab(p); + small_dealloc_offseted_slow(super, p, slab->get_meta().sizeclass); + } + SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( Superslab* super, void* p, sizeclass_t sizeclass) { diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h index 0d39db33f..766097b88 100644 --- a/src/mem/sizeclass.h +++ b/src/mem/sizeclass.h @@ -20,6 +20,7 @@ namespace snmalloc constexpr static size_t sizeclass_to_inverse_cache_friendly_mask(sizeclass_t sc); constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); + template static sizeclass_t size_to_sizeclass(size_t size); constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 5e067db42..830b534e2 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -110,9 +110,10 @@ namespace snmalloc return sizeclass_metadata.inverse_cache_friendly_mask[sizeclass]; } + template static inline sizeclass_t size_to_sizeclass(size_t size) { - if ((size - 1) <= (SLAB_SIZE - 1)) + if ((size - 1) <= (SLAB_SIZE - 1) || ASSUME_SMALL) { auto index = sizeclass_lookup_index(size); SNMALLOC_ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); diff --git a/src/override/malloc.cc b/src/override/malloc.cc index dc9f7a4ec..2c54455e1 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -69,6 +69,33 @@ extern "C" return ThreadAlloc::get_noncachable()->alloc(sz); } + SNMALLOC_EXPORT + void SNMALLOC_NAME_MANGLE(free_local_small)(void* ptr) + { + if (Alloc::small_local_dealloc(ptr)) + return; + ThreadAlloc::get_noncachable()->small_local_dealloc_slow(ptr); + } + + SNMALLOC_EXPORT + void* SNMALLOC_NAME_MANGLE(malloc_small)(size_t size) + { + return ThreadAlloc::get_noncachable()->small_alloc( + size); + } + + SNMALLOC_EXPORT + void* SNMALLOC_NAME_MANGLE(malloc_small_64)() + { + return ThreadAlloc::get_noncachable()->small_alloc(64); + } + + SNMALLOC_EXPORT + void* SNMALLOC_NAME_MANGLE(malloc_small_63)() + { + return ThreadAlloc::get_noncachable()->small_alloc(63); + } + SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)( MALLOC_USABLE_SIZE_QUALIFIER void* ptr) diff --git a/src/redirect/generate.cc b/src/redirect/generate.cc new file mode 100644 index 000000000..cccbf177a --- /dev/null +++ b/src/redirect/generate.cc @@ -0,0 +1,41 @@ +#include "snmalloc.h" + +#include +#include + +int main(int argc, char* argv[]) +{ + if (argc != 2) + { + std::cerr << "Call with output file name" << std::endl; + return 1; + } + + // open a file in write mode. + ofstream outfile; + outfile.open(argv[1]); + + for (size_t align = 0; align < 10; align++) + { + for (size_t size = 1024; size > 0; size -= 16) + { + auto asize = snmalloc::aligned_size(1ULL << align, size); + auto sizeclass = snmalloc::size_to_sizeclass(asize); + auto rsize = snmalloc::sizeclass_to_size(sizeclass); + if (rsize == size && align == 0) + { + outfile << "DEFINE_MALLOC_SIZE(__stack_alloc_small_" << size << "_" << align << ", " << size + << ");" << std::endl; + } + else + { + outfile << "REDIRECT_MALLOC_SIZE(__stack_alloc_small_" << size << "_" << align << ", __stack_alloc_small_" + << rsize << "_" << 0 << ");" << std::endl; + } + outfile << "GENERATE_FREE_SIZE(__stack_free_small_" << size << "_" << align << ");" << std::endl; + + } + } + + outfile.close(); +} \ No newline at end of file diff --git a/src/redirect/redirect.cc b/src/redirect/redirect.cc new file mode 100644 index 000000000..aadeeba56 --- /dev/null +++ b/src/redirect/redirect.cc @@ -0,0 +1,51 @@ + +#include "snmalloc.h" + +#define NAME(a) malloc_size_##a +#define STRINGIFY(a) a +#define NAME_STRING(a) NAME(a) + +#ifdef WIN32 +# define REDIRECT_MALLOC_SIZE(a, b) \ + extern "C" void* NAME(a)(); \ + __pragma(comment(linker, "/alternatename:##a=##b")) +#else +# define REDIRECT_MALLOC_SIZE(a, b) \ + __attribute__((alias(#b))) extern "C" void* a() +#endif + +#define DEFINE_MALLOC_SIZE(name, s) \ + extern "C" void* name() \ + { \ + return snmalloc::ThreadAlloc::get_noncachable()->template alloc(); \ + } + +extern "C" void free_local_small(void* ptr) +{ + if (snmalloc::Alloc::small_local_dealloc(ptr)) + return; + snmalloc::ThreadAlloc::get_noncachable()->small_local_dealloc_slow(ptr); +} + +#ifdef WIN32 +# define GENERATE_FREE_SIZE(a) \ + extern "C" void* NAME(a)(); \ + __pragma(comment(linker, "/alternatename:##a=free_local_small")) +#else +# define GENERATE_FREE_SIZE(a) \ + __attribute__((alias("free_local_small"))) extern "C" void* a() +#endif + +void* __stack_alloc_large(size_t size, size_t align) +{ + size_t asize = snmalloc::aligned_size(align, size); + return snmalloc::ThreadAlloc::get_noncachable()->alloc(asize); +} + +void __stack_free_large(void* ptr, size_t size, size_t align) +{ + size_t asize = snmalloc::aligned_size(align, size); + snmalloc::ThreadAlloc::get_noncachable()->dealloc(ptr, asize); +} + +#include "generated.cc" diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index b206cd599..e935068cd 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -116,6 +116,12 @@ void test_memalign(size_t size, size_t align, int err, bool null) check_result(size, align, p, err, null); } +void test_local(size_t size) +{ + for (int n = 0; n < 1000; n++) + our_free_local_small(our_malloc_small(size)); +} + int main(int argc, char** argv) { UNUSED(argc); @@ -125,6 +131,11 @@ int main(int argc, char** argv) constexpr int SUCCESS = 0; + for (size_t i = 1; i < SLAB_SIZE; i += 16) + { + test_local(i); + } + test_realloc(our_malloc(64), 4194304, SUCCESS, false); for (sizeclass_t sc = 0; sc < (SUPERSLAB_BITS + 4); sc++)