From 051288f125a82b27a384f76a411afced6e80e5bb Mon Sep 17 00:00:00 2001 From: Ian Ashworth Date: Mon, 11 Sep 2023 22:49:17 +0100 Subject: [PATCH 1/2] Rolled back my hibernation change of checking for "-52" - it fails on most saves I have tried but skipping an extra 112 bytes if the V9 check fails seems to do the trick. Added ArkSaveGame.StoredDataOffsets - read in during readBinaryHeader() As hinted by Alex, StructCustomItemDataRef was 2 bytes, 2nd byte represents index of stored data offset pairs. Added new class to hold the cryo, offset position and data in bytes to speed up read using parallel loop. CryoStoreData Re-worked readStoredBinaryObjects() to map and use memory based byte arrays when reading in cryo creature data. --- SavegameToolkit/ArkSavegame.cs | 240 +++++++++++------- .../Structs/StructCustomItemDataRef.cs | 11 +- SavegameToolkit/Types/CryoStoreData.cs | 16 ++ 3 files changed, 175 insertions(+), 92 deletions(-) create mode 100644 SavegameToolkit/Types/CryoStoreData.cs diff --git a/SavegameToolkit/ArkSavegame.cs b/SavegameToolkit/ArkSavegame.cs index f48f586..56d9466 100644 --- a/SavegameToolkit/ArkSavegame.cs +++ b/SavegameToolkit/ArkSavegame.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SavegameToolkit.Arrays; @@ -45,9 +46,9 @@ public class ArkSavegame : GameObjectContainerMixin, IConversionSupport [JsonProperty(Order = 7)] public override List Objects { get; } = new List(); - private int hibernationOffset; + public List> StoredDataOffsets = new List>(); - private int storedOffset; + private int hibernationOffset; private int nameTableOffset; @@ -64,7 +65,7 @@ public class ArkSavegame : GameObjectContainerMixin, IConversionSupport // Ark version 349.10 introduced two new unknown hibernation entries, the save version remained 9, though. Saving these is not supported. private int hibernationV9_34910Unknown1; private int hibernationV9_34910Unknown2; - + private int hibernationV9Check1; private int hibernationUnknown1; private int hibernationUnknown2; @@ -80,6 +81,11 @@ public class ArkSavegame : GameObjectContainerMixin, IConversionSupport public bool HasUnknownData { get; set; } private HashSet nameTableForWriteBinary; + private int hibernationV9Check2; + private int hibernationV9Check3; + private int hibernationV9Check4; + private int hibernationV9Check5; + private int hibernationV9Check6; #region readBinary @@ -133,19 +139,24 @@ private void readBinaryHeader(ArkArchive archive) if (SaveVersion > 10) { - storedOffset = (int)archive.ReadLong(); + + var storedOffset = archive.ReadLong(); var storedDataSize = archive.ReadLong(); + StoredDataOffsets.Add(new Tuple(storedOffset, storedDataSize)); var v11Unknown1 = archive.ReadLong(); //file size or some other pointer var v11Unknown2 = archive.ReadLong(); //0 + StoredDataOffsets.Add(new Tuple(v11Unknown1, v11Unknown2)); + + var v11Unknown3 = archive.ReadLong(); //file size or some other pointer var v11Unknown4 = archive.ReadLong(); //0 + StoredDataOffsets.Add(new Tuple(v11Unknown3, v11Unknown4)); + + var v11Unknown5 = archive.ReadLong(); //file size or some other pointer var v11Unknown6 = archive.ReadLong(); //0 - } - else - { - storedOffset = 0; + StoredDataOffsets.Add(new Tuple(v11Unknown5, v11Unknown6)); } hibernationOffset = archive.ReadInt(); @@ -359,7 +370,6 @@ private void readBinaryHibernation(ArkArchive archive, ReadingOptions options) archive.Position = hibernationOffset; - /* if (SaveVersion > 7) { hibernationV8Unknown1 = archive.ReadInt(); @@ -375,24 +385,32 @@ private void readBinaryHibernation(ArkArchive archive, ReadingOptions options) archive.DebugMessage("non-zero unknown V9 fields, expecting duplicated set per 349.10"); hibernationV9_34910Unknown1 = archive.ReadInt(); hibernationV9_34910Unknown2 = archive.ReadInt(); - if (!(hibernationV8Unknown1 == archive.ReadInt() - && hibernationV8Unknown2 == archive.ReadInt() - && hibernationV8Unknown3 == archive.ReadInt() - && hibernationV8Unknown4 == archive.ReadInt() - && hibernationV9_34910Unknown1 == archive.ReadInt() - && hibernationV9_34910Unknown2 == archive.ReadInt())) - { - throw new NotSupportedException("349.10 workaround for duplicate unknown hibernation bytes failed"); - } - } - */ - while (archive.Position < nameTableOffset) - { - //regardless of anything before the hibernated class count always seems to follow immediately after this byte in all save formats I have worked on. - if (archive.ReadSByte() == -52) + + hibernationV9Check1 = archive.ReadInt(); + hibernationV9Check2 = archive.ReadInt(); + hibernationV9Check3 = archive.ReadInt(); + hibernationV9Check4 = archive.ReadInt(); + hibernationV9Check5 = archive.ReadInt(); + hibernationV9Check6 = archive.ReadInt(); + + if (!(hibernationV8Unknown1 == hibernationV9Check1 + && hibernationV8Unknown2 == hibernationV9Check2 + && hibernationV8Unknown3 == hibernationV9Check3 + && hibernationV8Unknown4 == hibernationV9Check4 + && hibernationV9_34910Unknown1 == hibernationV9Check5 + && hibernationV9_34910Unknown2 == hibernationV9Check6)) { - break; + if (SaveVersion > 10) + { + //TODO:// more data reads - is it always 112 bytes? + archive.SkipBytes(112); + + } + else + { + throw new NotSupportedException("349.10 workaround for duplicate unknown hibernation bytes failed"); + } } } @@ -402,10 +420,9 @@ private void readBinaryHibernation(ArkArchive archive, ReadingOptions options) return; } - /* + hibernationUnknown1 = archive.ReadInt(); hibernationUnknown2 = archive.ReadInt(); - */ int hibernatedClassesCount = archive.ReadInt(); @@ -445,101 +462,148 @@ private void readBinaryStoredObjects(ArkArchive archive, ReadingOptions options) { if (!options.CryopodCreatures) return; - var inventoryContainers = Objects.Where(x => x.GetPropertyValue("MyInventoryComponent") != null).ToList(); - - var validStored = Objects + var inventoryContainers = Objects.AsParallel().Where(x => x.GetPropertyValue("MyInventoryComponent") != null).ToList(); + + var validStored = Objects.AsParallel() .Where(o => - (o.ClassName.Name.Contains("Cryopod") || o.ClassString.Contains("SoulTrap_") || o.ClassString.Contains("Vivarium_")) - && o.HasAnyProperty("CustomItemDatas") + (o.ClassString.Contains("Cryopod") || o.ClassString.Contains("SoulTrap_") || o.ClassString.Contains("Vivarium_")) + && o.GetTypedProperty("CustomItemDatas") != null ) - .ToArray(); + .ToList(); - foreach (var o in validStored) + List storedOffsets = new List(); + + validStored.ForEach(x => { - if (!(o.Properties.First(p => p.NameString == "CustomItemDatas") is PropertyArray customData)) continue; + if (!(x.Properties.First(p => p.NameString == "CustomItemDatas") is PropertyArray customData)) return; long cryoDataOffset = 0; + int dataFile = 1; + if ( archive.SaveVersion > 10 && customData.Value is ArkArrayStruct redirectors - && redirectors.All(x => x is StructCustomItemDataRef) - ) { + && redirectors.All(r => r is StructCustomItemDataRef) +) + { // TODO: probably best to: in cryos, take first entry. in souls, take second. cryoDataOffset = ((StructCustomItemDataRef)redirectors[0]).Position; + dataFile = ((StructCustomItemDataRef)redirectors[0]).StoreDataIndex; } - var creatureDataOffset = cryoDataOffset + storedOffset; - archive.Position = creatureDataOffset; - - ArkCryoStore cryoStore = new ArkCryoStore(archive); - if (!cryoStore.Objects.Any()) continue; - cryoStore.LoadProperties(archive); + storedOffsets.Add(new CryoStoreData() { ParentObject = x, Offset = cryoDataOffset, StoreDataIndex = dataFile }); - var dinoComponent = cryoStore.Objects[0]; - var dinoCharacterStatusComponent = cryoStore.Objects[1]; - var dinoInventoryComponent = cryoStore.Objects[2]; + }); + validStored = null; - //re-map and add properties as appropriate - if (dinoComponent == null) continue; + //sort and read bytes into memory between this and next offset + var sortedOffsets = storedOffsets.OrderBy(x => x.StoreDataIndex).ThenBy(x => x.Offset).ToList(); + for (int x = 0; x < sortedOffsets.Count; x++) + { + var storedObject = sortedOffsets[x]; + int storedOffsetIndex = storedObject.StoreDataIndex; + var offsetData = StoredDataOffsets[storedOffsetIndex]; - dinoComponent.IsInCryo = true; + var storedOffset = offsetData.Item1 + storedObject.Offset; + long nextOffset = offsetData.Item1 + offsetData.Item2; + if (x < sortedOffsets.Count - 1) + { + var nextItem = sortedOffsets[x + 1]; + if (nextItem.StoreDataIndex == storedObject.StoreDataIndex) + { + nextOffset = nextItem.Offset + offsetData.Item1; + } + } - // the tribe name is stored in `TamerString`, non-cryoed creatures have the property `TribeName` for that. - if (dinoComponent.GetPropertyValue("TribeName")?.Length == 0 && dinoComponent.GetPropertyValue("TamerString")?.Length > 0) - dinoComponent.Properties.Add(new PropertyString("TribeName", dinoComponent.GetPropertyValue("TamerString"))); + archive.Position = storedOffset; + int bytesToRead = (int)(nextOffset - storedOffset); + storedObject.Data = archive.ReadBytes(bytesToRead); + } + storedOffsets = new List(); - //get parent of cryopod owner inventory - var podParentRef = o.GetPropertyValue("OwnerInventory"); - if (podParentRef != null) + //load the cryo creature data in parallel to speed up read + Parallel.ForEach(sortedOffsets, o => + { + if (o.Data.Length == 0) return; + using (MemoryStream ms = new MemoryStream(o.Data)) { - var podParent = inventoryContainers.FirstOrDefault(x => x.GetPropertyValue("MyInventoryComponent")?.ObjectId == podParentRef.ObjectId); - - //determine if we need to re-team the podded animal - if (podParent != null) + using (ArkArchive storedArchive = new ArkArchive(ms)) { - dinoComponent.Parent = podParent; + ArkCryoStore cryoStore = new ArkCryoStore(storedArchive); + if (!cryoStore.Objects.Any()) return; + + cryoStore.LoadProperties(storedArchive); + + + var dinoComponent = cryoStore.Objects[0]; + var dinoCharacterStatusComponent = cryoStore.Objects[1]; + var dinoInventoryComponent = cryoStore.Objects[2]; + + //re-map and add properties as appropriate + if (dinoComponent == null) return; - int obTeam = dinoComponent.GetPropertyValue("TargetingTeam"); - int containerTeam = podParent.GetPropertyValue("TargetingTeam"); - if (obTeam != containerTeam) + dinoComponent.IsInCryo = true; + + // the tribe name is stored in `TamerString`, non-cryoed creatures have the property `TribeName` for that. + if (dinoComponent.GetPropertyValue("TribeName")?.Length == 0 && dinoComponent.GetPropertyValue("TamerString")?.Length > 0) + dinoComponent.Properties.Add(new PropertyString("TribeName", dinoComponent.GetPropertyValue("TamerString"))); + + //get parent of cryopod owner inventory + var podParentRef = o.ParentObject.GetPropertyValue("OwnerInventory"); + if (podParentRef != null) { - var propertyIndex = dinoComponent.Properties.FindIndex(i => i.NameString == "TargetingTeam"); - if (propertyIndex != -1) + var podParent = inventoryContainers.FirstOrDefault(x => x.GetPropertyValue("MyInventoryComponent")?.ObjectId == podParentRef.ObjectId); + + //determine if we need to re-team the podded animal + if (podParent != null) { - dinoComponent.Properties.RemoveAt(propertyIndex); - } - dinoComponent.Properties.Add(new PropertyInt("TargetingTeam", containerTeam)); + dinoComponent.Parent = podParent; + int obTeam = dinoComponent.GetPropertyValue("TargetingTeam"); + int containerTeam = podParent.GetPropertyValue("TargetingTeam"); + if (obTeam != containerTeam) + { + var propertyIndex = dinoComponent.Properties.FindIndex(i => i.NameString == "TargetingTeam"); + if (propertyIndex != -1) + { + dinoComponent.Properties.RemoveAt(propertyIndex); + } + dinoComponent.Properties.Add(new PropertyInt("TargetingTeam", containerTeam)); - if (dinoComponent.HasAnyProperty("TamingTeamID")) - { - dinoComponent.Properties.RemoveAt(dinoComponent.Properties.FindIndex(i => i.NameString == "TamingTeamID")); - dinoComponent.Properties.Add(new PropertyInt("TamingTeamID", containerTeam)); - } + if (dinoComponent.HasAnyProperty("TamingTeamID")) + { + dinoComponent.Properties.RemoveAt(dinoComponent.Properties.FindIndex(i => i.NameString == "TamingTeamID")); + dinoComponent.Properties.Add(new PropertyInt("TamingTeamID", containerTeam)); + } + + } + } } - } - } - if (dinoCharacterStatusComponent != null) - { - addObject(dinoCharacterStatusComponent, true); + if (dinoCharacterStatusComponent != null) + { + addObject(dinoCharacterStatusComponent, false); - var statusComponentRef = dinoComponent.GetTypedProperty("MyCharacterStatusComponent"); - statusComponentRef.Value.ObjectId = dinoCharacterStatusComponent.Id; + var statusComponentRef = dinoComponent.GetTypedProperty("MyCharacterStatusComponent"); + statusComponentRef.Value.ObjectId = dinoCharacterStatusComponent.Id; - } + } - if (dinoInventoryComponent != null) - { - addObject(dinoInventoryComponent, true); + if (dinoInventoryComponent != null) + { + addObject(dinoInventoryComponent, false); + + var inventoryComponentRef = dinoComponent.GetTypedProperty("MyInventoryComponent"); + inventoryComponentRef.Value.ObjectId = dinoInventoryComponent.Id; + } - var inventoryComponentRef = dinoComponent.GetTypedProperty("MyInventoryComponent"); - inventoryComponentRef.Value.ObjectId = dinoInventoryComponent.Id; + addObject(dinoComponent, false); + + } } + }); - addObject(dinoComponent, true); - } } diff --git a/SavegameToolkit/Structs/StructCustomItemDataRef.cs b/SavegameToolkit/Structs/StructCustomItemDataRef.cs index d822065..94bb572 100644 --- a/SavegameToolkit/Structs/StructCustomItemDataRef.cs +++ b/SavegameToolkit/Structs/StructCustomItemDataRef.cs @@ -10,18 +10,21 @@ namespace SavegameToolkit.Structs { public class StructCustomItemDataRef : StructBase { [JsonProperty(Order = 0)] - public short Unknown0 { get; private set; } + public byte Unknown0 { get; private set; } [JsonProperty(Order = 1)] - public long Position { get; private set; } + public byte StoreDataIndex{ get; private set; } [JsonProperty(Order = 2)] - public ObjectReference[] ObjectRefs { get; private set; } + public long Position { get; private set; } [JsonProperty(Order = 3)] + public ObjectReference[] ObjectRefs { get; private set; } + [JsonProperty(Order = 4)] public ObjectReference[] ClassRefs { get; private set; } public override void Init(ArkArchive archive) { // The first unknown field may be two fields - perhaps format version and archive index - Unknown0 = archive.ReadShort(); + Unknown0 = archive.ReadByte(); + StoreDataIndex = archive.ReadByte(); Position = archive.ReadLong(); ObjectRefs = new ObjectReference[archive.ReadInt()]; for (int index = 0; index < ObjectRefs.Length; index++) diff --git a/SavegameToolkit/Types/CryoStoreData.cs b/SavegameToolkit/Types/CryoStoreData.cs new file mode 100644 index 0000000..888a48e --- /dev/null +++ b/SavegameToolkit/Types/CryoStoreData.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SavegameToolkit.Types +{ + public class CryoStoreData + { + + public GameObject ParentObject { get; set; } + public int StoreDataIndex { get; set; } = 1; + public long Offset { get; set; } + public byte[] Data { get; set; } + + } +} From 000d03b726dc62a0bd70e9789b71b2f870c7726f Mon Sep 17 00:00:00 2001 From: Ian Ashworth Date: Tue, 12 Sep 2023 16:48:05 +0100 Subject: [PATCH 2/2] Generic loop to readin the stored offset data in V10+ header. --- SavegameToolkit/ArkSavegame.cs | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/SavegameToolkit/ArkSavegame.cs b/SavegameToolkit/ArkSavegame.cs index 56d9466..2d13b09 100644 --- a/SavegameToolkit/ArkSavegame.cs +++ b/SavegameToolkit/ArkSavegame.cs @@ -140,23 +140,12 @@ private void readBinaryHeader(ArkArchive archive) if (SaveVersion > 10) { - var storedOffset = archive.ReadLong(); - var storedDataSize = archive.ReadLong(); - StoredDataOffsets.Add(new Tuple(storedOffset, storedDataSize)); - - var v11Unknown1 = archive.ReadLong(); //file size or some other pointer - var v11Unknown2 = archive.ReadLong(); //0 - StoredDataOffsets.Add(new Tuple(v11Unknown1, v11Unknown2)); - - - var v11Unknown3 = archive.ReadLong(); //file size or some other pointer - var v11Unknown4 = archive.ReadLong(); //0 - StoredDataOffsets.Add(new Tuple(v11Unknown3, v11Unknown4)); - - - var v11Unknown5 = archive.ReadLong(); //file size or some other pointer - var v11Unknown6 = archive.ReadLong(); //0 - StoredDataOffsets.Add(new Tuple(v11Unknown5, v11Unknown6)); + //read in stored data file offset and sizes + for (int x = 0; x < 4; x++) + { + var storedOffset = new Tuple(archive.ReadLong(), archive.ReadLong()); + StoredDataOffsets.Add(storedOffset); + } } hibernationOffset = archive.ReadInt();