implement a feature to allocate an extra word ahead of each individua…

README.md

@@ -206,6 +206,15 @@ for mmtk-core, as well as compiling the fastpath for the VO bit.
 ```console
 $ VO_BIT=1 make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk
 ```
+### Extra header
+
+To support the `extra_header` (allocate an extra word) feature in mmtk-core, you can set the
+environment variable `EXTRA_HEADER=1` when building OpenJDK. This will set the feature
+for mmtk-core, as well as compiling the fastpath for the extra header.
+
+```console
+$ EXTRA_HEADER=1 make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk
+```
 
 ## Test
 

mmtk/Cargo.toml

@@ -46,6 +46,11 @@ nogc_no_zeroing = ["mmtk/nogc_no_zeroing"]
 # See README.
 vo_bit = ["mmtk/vo_bit"]
 
+# Use the env var EXTRA_HEADER=1 when building OpenJDK so the fastpath for allocating extra header will be compiled in.
+# See README.
+extra_header = ["mmtk/extra_header"]
+
+
 # This compile time constant places the mark bit in the header of the object instead of on the side.
 mark_bit_in_header = []
 

mmtk/src/lib.rs

@@ -188,3 +188,7 @@ lazy_static! {
 
 /// A counter tracking the total size of the `CODE_CACHE_ROOTS`.
 static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0);
+
+#[cfg(feature = "extra_header")]
+#[no_mangle]
+pub static MMTK_EXTRA_HEADER_BYTES: usize = <OpenJDK as VMBinding>::EXTRA_HEADER_BYTES;

openjdk/CompileThirdPartyHeap.gmk

@@ -27,6 +27,14 @@ ifeq ($(VO_BIT), 1)
   endif
 endif
 
+ifeq ($(EXTRA_HEADER), 1)
+  ifndef GC_FEATURES
+    GC_FEATURES=--features extra_header
+  else
+    GC_FEATURES:=$(strip $(GC_FEATURES))",extra_header"
+  endif
+endif
+
 ifeq ($(MARK_IN_HEADER), 1)
   ifndef GC_FEATURES
     GC_FEATURES=--features mark_bit_in_header
@@ -83,4 +91,8 @@ ifeq ($(VO_BIT), 1)
   JVM_CFLAGS += -DMMTK_ENABLE_VO_BIT
 endif
 
+ifeq ($(EXTRA_HEADER), 1)
+  JVM_CFLAGS += -DMMTK_ENABLE_EXTRA_HEADER
+endif
+
 $(BUILD_LIBJVM): $(LIB_MMTK)

openjdk/mmtk.h

@@ -23,6 +23,9 @@ extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS;
 extern const uintptr_t VO_BIT_ADDRESS;
 extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES;
 extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE;
+#ifdef MMTK_ENABLE_EXTRA_HEADER
+extern const size_t MMTK_EXTRA_HEADER_BYTES;
+#endif
 
 extern const char* get_mmtk_version();
 

openjdk/mmtkBarrierSetAssembler_x86.cpp

@@ -50,6 +50,9 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa
     assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized");
     size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes;
     size_t extra_header = 0;
+#ifdef MMTK_ENABLE_EXTRA_HEADER
+    extra_header = MMTK_EXTRA_HEADER_BYTES;
+#endif
     // fastpath, we only use default allocator
     Allocator allocator = AllocatorDefault;
     // We need to figure out which allocator we are using by querying MMTk.

openjdk/mmtkBarrierSetC2.cpp

@@ -82,6 +82,9 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
   assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized");
   size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes;
   size_t extra_header = 0;
+#ifdef MMTK_ENABLE_EXTRA_HEADER
+  extra_header = MMTK_EXTRA_HEADER_BYTES;
+#endif
   // We always use the default allocator.
   // But we need to figure out which allocator we are using by querying MMTk.
   AllocatorSelector selector = get_allocator_mapping(AllocatorDefault);

openjdk/mmtkMutator.cpp

@@ -21,7 +21,11 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) {
   // All allocations with size larger than max non-los bytes will get to this slowpath here.
   // We will use LOS for those.
   assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized");
-  if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) {
+  size_t extra_header = 0;
+#ifdef MMTK_ENABLE_EXTRA_HEADER
+  extra_header = MMTK_EXTRA_HEADER_BYTES;
+#endif
+  if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes - extra_header) {
     allocator = AllocatorLos;
   }