[UNTESTED] Fix integer overflow when reading late object properties i…

…n big files. Better V11 CustomItemDatas redirection support. (#6)

* Fix an int overflow when loading object properties beyond 2GB mark
* Provide SaveVersion through ArkArchive to rest of the code
* Read CustomItemDatas as native structs when version is higher than 11
Object and classes references are suspiciously missing from ArkCryoStore, which was crucial to figuring this out - thanks @miragedmuk.

SavegameToolkit/ArkArchive.cs

@@ -39,6 +39,8 @@ public class ArkArchive : IDisposable {
 
         private readonly byte[] smallByteBuffer = new byte[bufferSize];
 
+        public short SaveVersion { get; internal set; }
+
         /// <summary>
         /// Enable or disable the current nameTable
         /// </summary>
@@ -81,6 +83,7 @@ private ArkArchive(ArkArchive toClone, int size) {
             mbbReader = new BinaryReader(mbb);
             state = toClone.state;
             isSlice = true;
+            SaveVersion = toClone.SaveVersion;
         }
 
         public ArkArchive Clone() => new ArkArchive(this);

SavegameToolkit/ArkSavegame.cs

@@ -121,6 +121,8 @@ private void readBinaryHeader(ArkArchive archive)
         {
             SaveVersion = archive.ReadShort();
 
+            archive.SaveVersion = SaveVersion;
+
             if (SaveVersion < 5 || SaveVersion > 12)
             {
                 throw new NotSupportedException("Found unknown Version " + SaveVersion);
@@ -456,30 +458,14 @@ private void readBinaryStoredObjects(ArkArchive archive, ReadingOptions options)
             {
                 if (!(o.Properties.First(p => p.NameString == "CustomItemDatas") is PropertyArray customData)) continue;
 
-                var cryoDataOffset = 0;
-                if (customData.Value is ArkArrayUnknown dataByteArray)
-                {
-                    var dataBytes = dataByteArray.ToArray<byte>();
-
-                    if (dataBytes.Length > 0)
-                    {
-                        using (MemoryStream ms = new MemoryStream(dataBytes))
-                        {
-                            using (ArkArchive a = new ArkArchive(ms))
-                            {
-                                if (dataByteArray.Count >= 10) // only care about first 10 bytes, not sure sure what comes after that.
-                                {
-                                    var cryoUnknown1 = a.ReadShort();
-                                    var cryoUnknown2 = a.ReadInt();
-                                    cryoDataOffset = a.ReadInt();
-                                }
-                                else
-                                {
-                                    cryoDataOffset = 0;
-                                }
-                            }
-                        }
-                    }
+                long cryoDataOffset = 0;
+                if (
+                    archive.SaveVersion > 10
+                    && customData.Value is ArkArrayStruct redirectors
+                    && redirectors.All(x => x is StructCustomItemDataRef)
+                ) {
+                    // TODO: probably best to: in cryos, take first entry. in souls, take second.
+                    cryoDataOffset = ((StructCustomItemDataRef)redirectors[0]).Position;
                 }
 
                 var creatureDataOffset = cryoDataOffset + storedOffset;

SavegameToolkit/Arrays/ArkArrayStruct.cs

@@ -18,11 +18,30 @@ public class ArkArrayStruct : ArkArrayBase<IStruct> {
         private static readonly ArkName vector = ArkName.ConstantPlain("Vector");
 
         private static readonly ArkName linearColor = ArkName.ConstantPlain("LinearColor");
+
+        private static readonly ArkName customItemDatas = ArkName.ConstantPlain("CustomItemDatas");
 
         public override void Init(ArkArchive archive, PropertyArray property) {
             int size = archive.ReadInt();
 
             ArkName structType = StructRegistry.MapArrayNameToTypeName(property.Name);
+
+            // In versions 11 and above, CustomItemDatas properties seem to be redirected into separate archives. This
+            // is likely due to ARK writing different save file segments in memory first to circumvent internal 2GB
+            // limits on in-memory writes.
+            // The redirector seems to be implemented in a similar way to "plain" structs like Vectors, Quats, Colours.
+            // If the conditions match, let's take a detour here.
+            if (archive.SaveVersion > 10 && structType == null && property.Name == customItemDatas)
+            {
+                for (int n = 0; n < size; n++)
+                {
+                    StructCustomItemDataRef cidRef = new StructCustomItemDataRef();
+                    cidRef.Init(archive);
+                    Add(cidRef);
+                }
+                return;
+            }
+
             if (structType == null) {
                 if (size * 4 + 4 == property.DataSize) {
                     structType = color;

SavegameToolkit/GameObject.cs

@@ -280,7 +280,7 @@ private void readBinary(ArkArchive archive) {
         }
 
         public void LoadProperties(ArkArchive archive, GameObject next, int propertiesBlockOffset) {
-            int offset = propertiesBlockOffset + propertiesOffset;
+            long offset = propertiesBlockOffset + propertiesOffset;
             long nextOffset = propertiesBlockOffset + next?.propertiesOffset ?? archive.Limit;
 
             archive.Position = offset;

SavegameToolkit/Structs/StructCustomItemDataRef.cs

@@ -0,0 +1,59 @@
+using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using SavegameToolkit.Arrays;
+using SavegameToolkit.Types;
+
+namespace SavegameToolkit.Structs {
+
+    [JsonObject(MemberSerialization.OptIn)]
+    public class StructCustomItemDataRef : StructBase {
+
+        [JsonProperty(Order = 0)]
+        public short Unknown0 { get; private set; }
+        [JsonProperty(Order = 1)]
+        public long Position { get; private set; }
+        [JsonProperty(Order = 2)]
+        public ObjectReference[] ObjectRefs { get; private set; }
+        [JsonProperty(Order = 3)]
+        public ObjectReference[] ClassRefs { get; private set; }
+
+        public override void Init(ArkArchive archive)
+        {
+            // The first unknown field may be two fields - perhaps format version and archive index
+            Unknown0 = archive.ReadShort();
+            Position = archive.ReadLong();
+            ObjectRefs = new ObjectReference[archive.ReadInt()];
+            for (int index = 0; index < ObjectRefs.Length; index++)
+            {
+                ObjectRefs[index] = new ObjectReference(archive, 8);
+            }
+            ClassRefs = new ObjectReference[archive.ReadInt()];
+            for (int index = 0; index < ClassRefs.Length; index++)
+            {
+                ClassRefs[index] = new ObjectReference(archive, 8);
+            }
+        }
+
+        public override void Init(JObject node)
+        {
+            throw new NotImplementedException("JSON import of StructCustomItemDataRef has not been implemented");
+        }
+
+        public override void WriteJson(JsonTextWriter generator, WritingOptions writingOptions)
+        {
+            throw new NotImplementedException("JSON export of StructCustomItemDataRef has not been implemented");
+        }
+
+        public override void WriteBinary(ArkArchive archive)
+        {
+            throw new NotImplementedException("Binary export of StructCustomItemDataRef has not been implemented");
+        }
+
+        public override int Size(NameSizeCalculator nameSizer)
+        {
+            return sizeof(short) + sizeof(long) + sizeof(int) * 2 + ObjectRefs.Length * 8 + ClassRefs.Length * 8;
+        }
+    }
+
+}