Move heap ownership info from chunk to pagemap (#4371)

Prior to this commit, the chunk kept track of the heap ownership
info in the `chunk->actor` field. This worked but was not ideal
as noted in the `ponyint_heap_owner` function that this is a case
of false sharing where the chunk owner needs the chunk but
all other actors only need the chunk owner only. This led to lots
of pointer chasing as the pagemap was used to retrieve the chunk
pointerand then the chunk had to be loaded to retrieve the owning
actor pointer.

This commit removes the `chunk->actor` field further slimming down
both the `small_chunk` and `large_chunk` so both now fit within
32 bytes (on 64 bit architectures). The chunk ownership information
is now kept in the lowest level of the pagemap next to the chunk
information. This removes the previous pointer chasing because
the chunk owning actor pointer is retrieved at the same time as the
chunk pointer without needing to load the chunk itself. This is a
fairly standard tradeoff of memory for performance where we're now
storing more data in the pagemap to minimize pointer chasing. The
expectation is that this will have a net positive impact on
performance but no benchmarks have been run to validate this
assumption.

---------

Co-authored-by: Dipin Hora <[email protected]>
Co-authored-by: Sean T Allen <[email protected]>

packages/builtin_test/_test.pony

@@ -11,7 +11,7 @@ use @pony_alloc_final[Pointer[U8]](ctx: Pointer[None], size: USize)
 use @pony_exitcode[None](code: I32)
 use @pony_get_exitcode[I32]()
 use @pony_triggergc[None](ctx: Pointer[None])
-use @ponyint_pagemap_get[Pointer[None]](p: Pointer[None] tag)
+use @ponyint_pagemap_get_chunk[Pointer[None]](p: Pointer[None] tag)
 
 use "pony_test"
 use "collections"
@@ -617,12 +617,12 @@ class \nodoc\ iso _TestStringTrimInPlace is UnitTest
     space: USize = 0)
   =>
     let copy: String ref = orig.clone()
-    let pre_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let pre_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     copy.trim_in_place(from, to)
     h.assert_eq[String box](expected, copy)
     h.assert_eq[USize](space, copy.space())
     h.assert_eq[String box](expected, copy.clone()) // safe to clone
-    let post_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let post_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     if copy.space() == 0 then
       h.assert_eq[USize](0, post_trim_pagemap.usize())
     else
@@ -1480,11 +1480,11 @@ class \nodoc\ iso _TestArrayTrimInPlace is UnitTest
     space: USize = 0)
   =>
     let copy: Array[U8] ref = orig.clone()
-    let pre_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let pre_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     copy.trim_in_place(from, to)
     h.assert_eq[USize](space, copy.space())
     h.assert_array_eq[U8](expected, copy)
-    let post_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let post_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     if copy.space() == 0 then
       h.assert_eq[USize](0, post_trim_pagemap.usize())
     else

src/libponyrt/gc/gc.c

@@ -482,7 +482,8 @@ static void acq_or_rel_remote_object(pony_ctx_t* ctx, pony_actor_t* actor,
 void ponyint_gc_sendobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -492,8 +493,6 @@ void ponyint_gc_sendobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     send_local_object(ctx, p, t, mutability);
   else
@@ -503,7 +502,8 @@ void ponyint_gc_sendobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_recvobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -513,8 +513,6 @@ void ponyint_gc_recvobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     recv_local_object(ctx, p, t, mutability);
   else
@@ -524,7 +522,8 @@ void ponyint_gc_recvobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_markobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -534,8 +533,6 @@ void ponyint_gc_markobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     mark_local_object(ctx, chunk, p, t, mutability);
   else
@@ -545,7 +542,8 @@ void ponyint_gc_markobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_acquireobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -555,8 +553,6 @@ void ponyint_gc_acquireobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     acquire_local_object(ctx, p, t, mutability);
   else
@@ -566,7 +562,8 @@ void ponyint_gc_acquireobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_releaseobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -576,8 +573,6 @@ void ponyint_gc_releaseobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     release_local_object(ctx, p, t, mutability);
   else
@@ -665,7 +660,7 @@ void ponyint_gc_markimmutable(pony_ctx_t* ctx, gc_t* gc)
     {
       // Mark in our heap and recurse if it wasn't already marked.
       void* p = obj->address;
-      chunk_t* chunk = ponyint_pagemap_get(p);
+      chunk_t* chunk = ponyint_pagemap_get_chunk(p);
       mark_local_object(ctx, chunk, p, obj->type, PONY_TRACE_IMMUTABLE);
     }
   }

src/libponyrt/gc/objectmap.c

@@ -71,7 +71,7 @@ void ponyint_objectmap_sweep(objectmap_t* map)
 
     if(obj->rc > 0)
     {
-      chunk_t* chunk = ponyint_pagemap_get(p);
+      chunk_t* chunk = ponyint_pagemap_get_chunk(p);
       ponyint_heap_mark_shallow(chunk, p);
     } else {
       ponyint_objectmap_clearindex(map, i);

src/libponyrt/mem/heap.c

@@ -10,9 +10,6 @@
 
 typedef struct chunk_t
 {
-  // immutable
-  pony_actor_t* actor;
-
   // used for pointer tagging
   // bit 0 (lowest bit) for keeping track of chunk type (1 = small; 0 = large)
   // bit 1 for keeping track of chunks to be cleared
@@ -238,9 +235,9 @@ static char* get_m(chunk_t* chunk)
   return (char*)((uintptr_t)chunk->m & CHUNK_M_BITMASK);
 }
 
-static void large_pagemap(char* m, size_t size, chunk_t* chunk)
+static void large_pagemap(char* m, size_t size, chunk_t* chunk, pony_actor_t* actor)
 {
-  ponyint_pagemap_set_bulk(m, chunk, size);
+  ponyint_pagemap_set_bulk(m, chunk, actor, size);
 }
 
 static void maybe_clear_chunk(chunk_t* chunk)
@@ -326,7 +323,7 @@ static void destroy_small(small_chunk_t* chunk, uint32_t mark)
   final_small(chunk, FORCE_ALL_FINALISERS);
 
   char* m = get_m((chunk_t*)chunk);
-  ponyint_pagemap_set(m, NULL);
+  ponyint_pagemap_set(m, NULL, NULL);
   POOL_FREE(block_t, m);
   POOL_FREE(small_chunk_t, chunk);
 }
@@ -340,7 +337,7 @@ static void destroy_large(large_chunk_t* chunk, uint32_t mark)
   final_large(chunk, mark);
 
   char* m = get_m((chunk_t*)chunk);
-  large_pagemap(m, chunk->size, NULL);
+  large_pagemap(m, chunk->size, NULL, NULL);
 
   if(m != NULL)
     ponyint_pool_free_size(chunk->size, m);
@@ -597,7 +594,6 @@ void* ponyint_heap_alloc_small(pony_actor_t* actor, heap_t* heap,
     }
   } else {
     small_chunk_t* n = (small_chunk_t*) POOL_ALLOC(small_chunk_t);
-    n->base.actor = actor;
     n->base.m = (char*) POOL_ALLOC(block_t);
     set_small_chunk_size(n, sizeclass);
 #ifdef USE_RUNTIMESTATS
@@ -617,7 +613,7 @@ void* ponyint_heap_alloc_small(pony_actor_t* actor, heap_t* heap,
 
     set_chunk_needs_clearing((chunk_t*)n);
 
-    ponyint_pagemap_set(get_m((chunk_t*)n), (chunk_t*)n);
+    ponyint_pagemap_set(get_m((chunk_t*)n), (chunk_t*)n, actor);
 
     heap->small_free[sizeclass] = n;
     chunk = n;
@@ -648,7 +644,6 @@ void* ponyint_heap_alloc_large(pony_actor_t* actor, heap_t* heap, size_t size,
   size = ponyint_pool_adjust_size(size);
 
   large_chunk_t* chunk = (large_chunk_t*) POOL_ALLOC(large_chunk_t);
-  chunk->base.actor = actor;
   chunk->size = size;
   chunk->base.m = (char*) ponyint_pool_alloc_size(size);
 #ifdef USE_RUNTIMESTATS
@@ -666,7 +661,7 @@ void* ponyint_heap_alloc_large(pony_actor_t* actor, heap_t* heap, size_t size,
   // note if a finaliser needs to run or not
   set_large_chunk_finaliser(chunk, (track_finalisers_mask & 1));
 
-  large_pagemap(get_m((chunk_t*)chunk), size, (chunk_t*)chunk);
+  large_pagemap(get_m((chunk_t*)chunk), size, (chunk_t*)chunk, actor);
 
   chunk->next = heap->large;
   heap->large = chunk;
@@ -685,7 +680,7 @@ void* ponyint_heap_realloc(pony_actor_t* actor, heap_t* heap, void* p,
   actor->actorstats.heap_realloc_counter++;
 #endif
 
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  chunk_t* chunk = ponyint_pagemap_get_chunk(p);
 
   // We can't realloc memory that wasn't pony_alloc'ed since we can't know how
   // much to copy from the previous location.
@@ -908,18 +903,6 @@ void ponyint_heap_endgc(heap_t* heap
     heap->next_gc = heap_initialgc;
 }
 
-pony_actor_t* ponyint_heap_owner(chunk_t* chunk)
-{
-  // FIX: false sharing
-  // reading from something that will never be written
-  // but is on a cache line that will often be written
-  // called during tracing
-  // actual chunk only needed for GC tracing
-  // all other tracing only needs the owner
-  // so the owner needs the chunk and everyone else just needs the owner
-  return chunk->actor;
-}
-
 size_t ponyint_heap_size(chunk_t* chunk)
 {
   if(get_chunk_is_small_chunk(chunk))

src/libponyrt/mem/heap.h

@@ -102,8 +102,6 @@ void ponyint_heap_endgc(heap_t* heap
   );
 #endif
 
-pony_actor_t* ponyint_heap_owner(chunk_t* chunk);
-
 size_t ponyint_heap_size(chunk_t* chunk);
 
 #ifdef USE_RUNTIMESTATS