Implement SBE serialization of ABS message types.

onyx-platform · Jan 23, 2017 · 7fe4745 · 7fe4745
1 parent 29ab5a9
commit 7fe4745
Show file tree

Hide file tree

Showing 33 changed files with 3,578 additions and 107 deletions.
diff --git a/project.clj b/project.clj
@@ -1,4 +1,4 @@
-(defproject org.onyxplatform/onyx "0.10.0-technical-preview-4"
+(defproject org.onyxplatform/onyx "0.10.0-technical-preview-5"
   :description "Distributed, masterless, high performance, fault tolerant data processing for Clojure"
   :url "https://github.com/onyx-platform/onyx"
   :license {:name "Eclipse Public License"
@@ -35,7 +35,7 @@
                                com.cemerick/austin]]
                  [prismatic/schema "1.0.5"]
                  [log4j/log4j "1.2.17"]
-                 [uk.co.real-logic/sbe-all "1.5.5"]
+                 ;[uk.co.real-logic/sbe-all "1.5.5"]
                  [com.amazonaws/aws-java-sdk-s3 "1.11.58"]
                  [clj-tuple "0.2.2"]
                  [clj-fuzzy "0.3.1" :exclusions [org.clojure/clojurescript]]]
@@ -48,6 +48,7 @@
                        "-XX:+UnlockCommercialFeatures"
                        "-XX:+FlightRecorder"
                        "-XX:StartFlightRecording=duration=1080s,filename=recording.jfr"]
+  :java-source-paths ["src-java"]
   :profiles {:dev {:global-vars {*warn-on-reflection* true}
                    :dependencies [[org.clojure/tools.nrepl "0.2.11"]
                                   [clojure-future-spec "1.9.0-alpha14"]

diff --git a/resources/onyx-schema.xml b/resources/onyx-schema.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe"
+                   package="onyx.serialization"
+                   id="1"
+                   version="0"
+                   semanticVersion="5.2"
+                   description="Example schema"
+                   byteOrder="littleEndian">
+    <types>
+        <composite name="messageHeader" description="Message identifiers and length of message root">
+            <type name="blockLength" primitiveType="uint16"/>
+            <type name="templateId" primitiveType="uint16"/>
+            <type name="schemaId" primitiveType="uint16"/>
+            <type name="version" primitiveType="uint16"/>
+        </composite>
+        <composite name="groupSizeEncoding" description="Repeating group dimensions">
+            <type name="blockLength" primitiveType="uint16"/>
+            <type name="numInGroup" primitiveType="uint16"/>
+        </composite>
+        <composite name="varDataEncoding">
+            <type name="length" primitiveType="uint32" maxValue="1073741824"/>
+            <type name="varData" primitiveType="int8"/>
+        </composite>
+        <composite name="keyValueEncoding">
+            <type name="length" primitiveType="uint16" maxValue="65534"/>
+            <type name="varData" primitiveType="int8"/>
+        </composite>
+        <composite name="valueValueEncoding">
+            <type name="length" primitiveType="uint16" maxValue="65534"/>
+            <type name="varData" primitiveType="int8"/>
+        </composite>
+    </types>
+    <sbe:message name="Message" id="1" description="OnyxMessages">
+        <field name="replicaVersion" id="2" type="uint64"/>
+        <field name="destId" id="3" type="uint16"/>
+	<group name="segments" id="4" dimensionType="groupSizeEncoding">
+	  <data name="segmentBytes" id="5" type="varDataEncoding"/>
+        </group>
+    </sbe:message>
+    <sbe:message name="Barrier" id="15" description="Barrier">
+        <field name="replicaVersion" id="16" type="uint64"/>
+        <field name="epoch" id="17" type="uint64"/>
+        <field name="destId" id="18" type="uint16"/>
+        <data name="payloadBytes" id="19" type="varDataEncoding"/>
+    </sbe:message>
+    <sbe:message name="DynamicMap" id="25" description="OnyxMap">
+	<field name="keyType" id="26" type="int8"/>
+	<field name="valueType" id="27" type="int8"/>
+        <data name="key" id="28" type="keyValueEncoding"/>
+        <data name="value" id="29" type="valueValueEncoding"/>
+    </sbe:message>
+</sbe:messageSchema>
diff --git a/scripts/generate_sbe.sh b/scripts/generate_sbe.sh
@@ -0,0 +1,2 @@
+curl -o sbe-all.jar -L "http://search.maven.org/remotecontent?filepath=uk/co/real-logic/sbe-all/1.5.5/sbe-all-1.5.5.jar"
+java -jar sbe-all.jar resources/onyx-schema.xml
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		curl -o sbe-all.jar -L "http://search.maven.org/remotecontent?filepath=uk/co/real-logic/sbe-all/1.5.5/sbe-all-1.5.5.jar"
		java -jar sbe-all.jar resources/onyx-schema.xml