Deterministic serialisation for cross binary communication

.release-notes/4567.md

@@ -0,0 +1,3 @@
+## Deterministic cerealisation for cross binary communication
+
+Pony built-in serialisation can now be used between binaries compiled with the same version of the pony compiler (this would likely result in segfaults previously). Cross binary serialisation will only work for binaries of the same bit width (32 bit vs 64 bit), data model (ilp32, lp64, or llp64), and endianness (big endian or little endian) but is not limited to a single platform (for example: one can mix and match x86_64 linux and aarch64 linux because they have the same bitwidth, data model, and endianness).

src/libponyc/ast/ast.c

@@ -2013,6 +2013,7 @@ static pony_type_t ast_signature_pony =
   sizeof(ast_signature_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   ast_signature_serialise_trace,
@@ -2092,6 +2093,7 @@ static pony_type_t ast_nominal_pkg_id_signature_pony =
   sizeof(ast_signature_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   ast_nominal_pkg_id_signature_serialise_trace,
@@ -2355,6 +2357,7 @@ static pony_type_t ast_pony =
   sizeof(ast_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   ast_serialise_trace,

src/libponyc/ast/source.c

@@ -114,6 +114,7 @@ static pony_type_t source_pony =
   sizeof(source_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   source_serialise_trace,

src/libponyc/ast/stringtab.c

@@ -134,6 +134,7 @@ static __pony_thread_local struct _pony_type_t string_pony =
   0,
   0,
   0,
+  0,
   NULL,
   NULL,
   NULL,
@@ -220,6 +221,7 @@ static pony_type_t strlist_pony =
   sizeof(strlist_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   strlist_serialise_trace,

src/libponyc/ast/symtab.c

@@ -389,6 +389,7 @@ static pony_type_t symbol_pony =
   sizeof(symbol_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   symbol_serialise_trace,

src/libponyc/ast/token.c

@@ -404,6 +404,7 @@ static pony_type_t token_signature_pony =
   sizeof(token_signature_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   token_signature_serialise_trace,
@@ -458,6 +459,7 @@ static pony_type_t token_docstring_signature_pony =
   sizeof(token_signature_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   token_docstring_signature_serialise_trace,
@@ -546,6 +548,7 @@ static pony_type_t token_pony =
   sizeof(token_t),
   0,
   0,
+  0,
   NULL,
   NULL,
   token_serialise_trace,

src/libponyc/codegen/codegen.c

@@ -15,6 +15,7 @@
 #include "../../libponyrt/mem/pool.h"
 #include "ponyassert.h"
 
+#include <blake2.h>
 #include <platform.h>
 #include <llvm-c/DebugInfo.h>
 #include <llvm-c/Initialization.h>
@@ -202,6 +203,13 @@ static void init_runtime(compile_t* c)
   c->msg_type = LLVMStructCreateNamed(c->context, "__message");
   LLVMStructSetBody(c->msg_type, params, 2, false);
 
+  // descriptor_offset_lookup
+  // uint32_t (*)(size_t)
+  params[0] = target_is_ilp32(c->opt->triple) ? c->i32 : c->i64;
+  c->descriptor_offset_lookup_type = LLVMFunctionType(c->i32, params, 1, false);
+  c->descriptor_offset_lookup_fn =
+    LLVMPointerType(c->descriptor_offset_lookup_type, 0);
+
   // trace
   // void (*)(i8*, __object*)
   params[0] = c->ptr;
@@ -856,6 +864,21 @@ bool codegen_pass_init(pass_opt_t* opt)
   else
     opt->cpu = LLVMGetHostCPUName();
 
+  opt->serialise_id_hash_key = (unsigned char*)ponyint_pool_alloc_size(16);
+
+  const char* version = "pony-" PONY_VERSION;
+  const char* data_model = target_is_ilp32(opt->triple) ? "ilp32" : (target_is_lp64(opt->triple) ? "lp64" : (target_is_llp64(opt->triple) ? "llp64" : "unknown"));
+  const char* endian = target_is_bigendian(opt->triple) ? "be" : "le";
+
+  printbuf_t* target_version_buf = printbuf_new();
+  printbuf(target_version_buf, "%s-%s-%s", version, data_model, endian);
+
+  int status = blake2b(opt->serialise_id_hash_key, 16, target_version_buf->m, target_version_buf->offset, NULL, 0);
+  (void)status;
+  pony_assert(status == 0);
+
+  printbuf_free(target_version_buf);
+
   return true;
 }
 
@@ -872,6 +895,9 @@ void codegen_pass_cleanup(pass_opt_t* opt)
   opt->triple = NULL;
   opt->cpu = NULL;
   opt->features = NULL;
+
+  ponyint_pool_free_size(16, opt->serialise_id_hash_key);
+  opt->serialise_id_hash_key = NULL;
 }
 
 bool codegen(ast_t* program, pass_opt_t* opt)

src/libponyc/codegen/codegen.h

@@ -172,6 +172,7 @@ typedef struct compile_t
   LLVMValueRef primitives_init;
   LLVMValueRef primitives_final;
   LLVMValueRef desc_table;
+  LLVMValueRef desc_table_offset_lookup_fn;
   LLVMValueRef numeric_sizes;
 
   LLVMTypeRef void_type;
@@ -187,6 +188,8 @@ typedef struct compile_t
 
   LLVMTypeRef ptr;
   LLVMTypeRef descriptor_type;
+  LLVMTypeRef descriptor_offset_lookup_type;
+  LLVMTypeRef descriptor_offset_lookup_fn;
   LLVMTypeRef field_descriptor;
   LLVMTypeRef object_type;
   LLVMTypeRef msg_type;

src/libponyc/codegen/gendesc.c

@@ -12,24 +12,25 @@
 
 #define DESC_ID 0
 #define DESC_SIZE 1
-#define DESC_FIELD_COUNT 2
-#define DESC_FIELD_OFFSET 3
-#define DESC_INSTANCE 4
-#define DESC_TRACE 5
-#define DESC_SERIALISE_TRACE 6
-#define DESC_SERIALISE 7
-#define DESC_DESERIALISE 8
-#define DESC_CUSTOM_SERIALISE_SPACE 9
-#define DESC_CUSTOM_DESERIALISE 10
-#define DESC_DISPATCH 11
-#define DESC_FINALISE 12
-#define DESC_EVENT_NOTIFY 13
-#define DESC_MIGHT_REFERENCE_ACTOR 14
-#define DESC_TRAITS 15
-#define DESC_FIELDS 16
-#define DESC_VTABLE 17
-
-#define DESC_LENGTH 18
+#define DESC_SERIALISEID 2
+#define DESC_FIELD_COUNT 3
+#define DESC_FIELD_OFFSET 4
+#define DESC_INSTANCE 5
+#define DESC_TRACE 6
+#define DESC_SERIALISE_TRACE 7
+#define DESC_SERIALISE 8
+#define DESC_DESERIALISE 9
+#define DESC_CUSTOM_SERIALISE_SPACE 10
+#define DESC_CUSTOM_DESERIALISE 11
+#define DESC_DISPATCH 12
+#define DESC_FINALISE 13
+#define DESC_EVENT_NOTIFY 14
+#define DESC_MIGHT_REFERENCE_ACTOR 15
+#define DESC_TRAITS 16
+#define DESC_FIELDS 17
+#define DESC_VTABLE 18
+
+#define DESC_LENGTH 19
 
 static LLVMValueRef make_unbox_function(compile_t* c, reach_type_t* t,
   reach_method_t* m)
@@ -330,6 +331,7 @@ void gendesc_basetype(compile_t* c, LLVMTypeRef desc_type)
 
   params[DESC_ID] = c->i32;
   params[DESC_SIZE] = c->i32;
+  params[DESC_SERIALISEID] = target_is_ilp32(c->opt->triple) ? c->i32 : c->i64;
   params[DESC_FIELD_COUNT] = c->i32;
   params[DESC_FIELD_OFFSET] = c->i32;
   params[DESC_INSTANCE] = c->ptr;
@@ -377,6 +379,7 @@ void gendesc_type(compile_t* c, reach_type_t* t)
 
   params[DESC_ID] = c->i32;
   params[DESC_SIZE] = c->i32;
+  params[DESC_SERIALISEID] = target_is_ilp32(c->opt->triple) ? c->i32 : c->i64;
   params[DESC_FIELD_COUNT] = c->i32;
   params[DESC_FIELD_OFFSET] = c->i32;
   params[DESC_INSTANCE] = c->ptr;
@@ -414,6 +417,7 @@ void gendesc_init(compile_t* c, reach_type_t* t)
   LLVMValueRef args[DESC_LENGTH];
   args[DESC_ID] = LLVMConstInt(c->i32, t->type_id, false);
   args[DESC_SIZE] = LLVMConstInt(c->i32, c_t->abi_size, false);
+  args[DESC_SERIALISEID] = LLVMConstInt(target_is_ilp32(c->opt->triple) ? c->i32 : c->i64, t->serialise_id, false);
   args[DESC_FIELD_COUNT] = make_field_count(c, t);
   args[DESC_FIELD_OFFSET] = make_field_offset(c, t);
   args[DESC_INSTANCE] = make_desc_ptr(c, c_t->instance);
@@ -478,6 +482,58 @@ void gendesc_table(compile_t* c)
   ponyint_pool_free_size(size, args);
 }
 
+void gendesc_table_lookup(compile_t* c)
+{
+  reach_type_t* t;
+  size_t i = HASHMAP_BEGIN;
+
+  LLVMValueRef desc_lkp_fn = codegen_addfun(c, "__DescOffsetLookupFn",
+    c->descriptor_offset_lookup_type, false);
+  codegen_startfun(c, desc_lkp_fn, NULL, NULL, NULL, false);
+  LLVMSetFunctionCallConv(desc_lkp_fn, LLVMCCallConv);
+  LLVMSetLinkage(desc_lkp_fn, LLVMExternalLinkage);
+
+  LLVMBasicBlockRef unreachable = codegen_block(c, "unreachable");
+
+  // Read the serialise ID.
+  LLVMValueRef serialise_id = LLVMGetParam(desc_lkp_fn, 0);
+
+  // switch based on serialise_id
+  LLVMValueRef serialise_switch = LLVMBuildSwitch(c->builder, serialise_id, unreachable, 0);
+
+  // the default case is unreachable unless something major has gone wrong
+  LLVMPositionBuilderAtEnd(c->builder, unreachable);
+
+  LLVMValueRef ret = LLVMConstInt(c->i32, (uint32_t)-1, false);
+  LLVMBuildRet(c->builder, ret);
+
+  while((t = reach_types_next(&c->reach->types, &i)) != NULL)
+  {
+    if(t->is_trait || (t->underlying == TK_STRUCT))
+      continue;
+
+    pony_assert(t->serialise_id != (uint64_t)-1);
+
+    LLVMBasicBlockRef type_block = codegen_block(c,
+      genname_type_with_id(t->name, t->serialise_id));
+
+    LLVMAddCase(serialise_switch, LLVMConstInt(target_is_ilp32(c->opt->triple) ? c->i32 : c->i64, t->serialise_id, false),
+      type_block);
+
+    LLVMPositionBuilderAtEnd(c->builder, type_block);
+
+    ret = LLVMConstInt(c->i32, t->type_id, false);
+    LLVMBuildRet(c->builder, ret);
+  }
+
+  // Mark the default case as unreachable.
+  LLVMPositionBuilderAtEnd(c->builder, unreachable);
+
+  codegen_finishfun(c);
+
+  c->desc_table_offset_lookup_fn = make_desc_ptr(c, desc_lkp_fn);
+}
+
 static LLVMValueRef desc_field(compile_t* c, LLVMValueRef desc, int index)
 {
   LLVMTypeRef field_type = LLVMStructGetTypeAtIndex(c->descriptor_type, index);

src/libponyc/codegen/gendesc.h

@@ -14,6 +14,8 @@ void gendesc_init(compile_t* c, reach_type_t* t);
 
 void gendesc_table(compile_t* c);
 
+void gendesc_table_lookup(compile_t* c);
+
 LLVMValueRef gendesc_fetch(compile_t* c, LLVMValueRef object);
 
 LLVMValueRef gendesc_typeid(compile_t* c, LLVMValueRef desc);

src/libponyc/codegen/genexe.c

@@ -52,14 +52,16 @@ static LLVMValueRef make_lang_features_init(compile_t* c)
     boolean = c->i8;
 
   uint32_t desc_table_size = reach_max_type_id(c->reach);
+  LLVMValueRef desc_table_lookup_fn = c->desc_table_offset_lookup_fn;
 
-  LLVMTypeRef f_params[4];
+  LLVMTypeRef f_params[5];
   f_params[0] = boolean;
   f_params[1] = boolean;
   f_params[2] = c->ptr;
   f_params[3] = c->intptr;
+  f_params[4] = c->descriptor_offset_lookup_fn;
 
-  LLVMTypeRef lfi_type = LLVMStructTypeInContext(c->context, f_params, 4,
+  LLVMTypeRef lfi_type = LLVMStructTypeInContext(c->context, f_params, 5,
     false);
 
   LLVMBasicBlockRef this_block = LLVMGetInsertBlock(c->builder);
@@ -89,6 +91,10 @@ static LLVMValueRef make_lang_features_init(compile_t* c)
   LLVMBuildStore(c->builder, LLVMConstInt(c->intptr, desc_table_size, false),
     field);
 
+  field = LLVMBuildStructGEP2(c->builder, lfi_type, lfi_object, 4, "");
+  LLVMBuildStore(c->builder, LLVMBuildBitCast(c->builder, desc_table_lookup_fn,
+    c->descriptor_offset_lookup_fn, ""), field);
+
   return lfi_object;
 }
 

src/libponyc/codegen/genname.c

@@ -199,3 +199,10 @@ const char* genname_program_fn(const char* program, const char* name)
 {
   return stringtab_two(program, name);
 }
+
+const char* genname_type_with_id(const char* type, uint64_t type_id)
+{
+  printbuf_t* buf = printbuf_new();
+  printbuf(buf, "%s_%" PRIu64, type, type_id);
+  return stringtab_buf(buf);
+}

src/libponyc/codegen/genname.h

@@ -42,6 +42,8 @@ const char* genname_unsafe(const char* name);
 
 const char* genname_program_fn(const char* program, const char* name);
 
+const char* genname_type_with_id(const char* type, uint64_t type_id);
+
 PONY_EXTERN_C_END
 
 #endif

src/libponyc/codegen/genprim.c

@@ -776,7 +776,7 @@ void genprim_array_serialise(compile_t* c, reach_type_t* t)
   LLVMValueRef offset_addr = LLVMBuildInBoundsGEP2(c->builder, c->i8, addr,
     &offset, 1, "");
 
-  genserialise_typeid(c, t, offset_addr);
+  genserialise_serialiseid(c, t, offset_addr);
 
   // Don't serialise our contents if we are opaque.
   LLVMBasicBlockRef body_block = codegen_block(c, "body");
@@ -890,7 +890,7 @@ void genprim_array_deserialise(compile_t* c, reach_type_t* t)
 
   LLVMValueRef ctx = LLVMGetParam(c_t->deserialise_fn, 0);
   LLVMValueRef object = LLVMGetParam(c_t->deserialise_fn, 1);
-  gendeserialise_typeid(c, c_t, object);
+  gendeserialise_serialiseid(c, c_t, object);
 
   // Deserialise the array contents.
   LLVMValueRef alloc = field_value(c, c_t->structure, object, 2);
@@ -1010,7 +1010,7 @@ void genprim_string_serialise(compile_t* c, reach_type_t* t)
   LLVMValueRef offset_addr = LLVMBuildInBoundsGEP2(c->builder, c->i8, addr,
     &offset, 1, "");
 
-  genserialise_typeid(c, t, offset_addr);
+  genserialise_serialiseid(c, t, offset_addr);
 
   // Don't serialise our contents if we are opaque.
   LLVMBasicBlockRef body_block = codegen_block(c, "body");
@@ -1069,7 +1069,7 @@ void genprim_string_deserialise(compile_t* c, reach_type_t* t)
   LLVMValueRef ctx = LLVMGetParam(c_t->deserialise_fn, 0);
   LLVMValueRef object = LLVMGetParam(c_t->deserialise_fn, 1);
 
-  gendeserialise_typeid(c, c_t, object);
+  gendeserialise_serialiseid(c, c_t, object);
 
   // Deserialise the string contents.
   LLVMValueRef alloc = field_value(c, c_t->structure, object, 2);