diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index d9bf18493..f267f4410 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -167,7 +167,7 @@ namespace snmalloc * passed to the core allocator. */ template - SNMALLOC_SLOW_PATH capptr::Alloc alloc_not_small(size_t size) + SNMALLOC_FAST_PATH capptr::Alloc alloc_not_small_fast(size_t size) { if (size == 0) { @@ -206,6 +206,12 @@ namespace snmalloc }); } + template + SNMALLOC_SLOW_PATH capptr::Alloc alloc_not_small(size_t size) + { + return alloc_not_small_fast(size); + } + template SNMALLOC_FAST_PATH capptr::Alloc small_alloc(size_t size) { @@ -241,6 +247,12 @@ namespace snmalloc domesticate, size, slowpath); } + template + SNMALLOC_SLOW_PATH capptr::Alloc small_alloc_slow(size_t size) + { + return small_alloc(size); + } + /** * Send all remote deallocation to other threads. */ @@ -416,8 +428,8 @@ namespace snmalloc /** * Allocate memory of a dynamically known size. */ - template - SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) + template + SNMALLOC_FAST_PATH ALLOCATOR void* alloc_hinted(size_t size) { #ifdef SNMALLOC_PASS_THROUGH // snmalloc guarantees a lot of alignment, so we can depend on this @@ -429,27 +441,46 @@ namespace snmalloc memset(result, 0, size); return result; #else - // Perform the - 1 on size, so that zero wraps around and ends up on - // slow path. - if (SNMALLOC_LIKELY( - (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) + if constexpr ( + (size_hint - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1)) { - // Small allocations are more likely. Improve - // branch prediction by placing this case first. - return capptr_reveal(small_alloc(size)); + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + if (SNMALLOC_LIKELY( + (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) + { + return capptr_reveal(small_alloc(size)); + } + + return capptr_reveal(alloc_not_small(size)); } + else + { + if (SNMALLOC_UNLIKELY( + (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) + { + return capptr_reveal(small_alloc_slow(size)); + } - return capptr_reveal(alloc_not_small(size)); + return capptr_reveal(alloc_not_small_fast(size)); + } #endif } + template + SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) + { + // Small allocations are more likely + return alloc_hinted<1, zero_mem>(size); + } + /** * Allocate memory of a statically known size. */ template SNMALLOC_FAST_PATH ALLOCATOR void* alloc() { - return alloc(size); + return alloc_hinted(size); } /*