Skip to content

Commit

Permalink
Various changes following testing. Added support for files larger tha…
Browse files Browse the repository at this point in the history 
…n 4GB. (#7)

* Rolled back my hibernation change of checking for "-52" - it fails on most saves I have tried but skipping an extra 112 bytes if the V9 check fails seems to do the trick.

Added ArkSaveGame.StoredDataOffsets - read in during readBinaryHeader()

As hinted by Alex, StructCustomItemDataRef was 2 bytes, 2nd byte represents index of stored data offset pairs.

Added new class to hold the cryo, offset position and data in bytes to speed up read using parallel loop. CryoStoreData

Re-worked readStoredBinaryObjects() to map and use memory based byte arrays when reading in cryo creature data.

* Generic loop to readin the stored offset data in V10+ header.

---------

Co-authored-by: cadon <[email protected]>
  • Loading branch information
@miragedmuk @cadon
miragedmuk and cadon authored Sep 12, 2023
1 parent 6665ccc commit 7e0b4a2
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 100 deletions.
244 changes: 148 additions & 96 deletions SavegameToolkit/ArkSavegame.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using SavegameToolkit.Arrays;
Expand Down Expand Up @@ -45,9 +46,9 @@ public class ArkSavegame : GameObjectContainerMixin, IConversionSupport
[JsonProperty(Order = 7)]
public override List<GameObject> Objects { get; } = new List<GameObject>();

private int hibernationOffset;
public List<Tuple<long, long>> StoredDataOffsets = new List<Tuple<long, long>>();

private int storedOffset;
private int hibernationOffset;

private int nameTableOffset;

Expand All @@ -64,7 +65,7 @@ public class ArkSavegame : GameObjectContainerMixin, IConversionSupport
// Ark version 349.10 introduced two new unknown hibernation entries, the save version remained 9, though. Saving these is not supported.
private int hibernationV9_34910Unknown1;
private int hibernationV9_34910Unknown2;

private int hibernationV9Check1;
private int hibernationUnknown1;

private int hibernationUnknown2;
Expand All @@ -80,6 +81,11 @@ public class ArkSavegame : GameObjectContainerMixin, IConversionSupport
public bool HasUnknownData { get; set; }

private HashSet<string> nameTableForWriteBinary;
private int hibernationV9Check2;
private int hibernationV9Check3;
private int hibernationV9Check4;
private int hibernationV9Check5;
private int hibernationV9Check6;

#region readBinary

Expand Down Expand Up @@ -133,19 +139,13 @@ private void readBinaryHeader(ArkArchive archive)

if (SaveVersion > 10)
{
storedOffset = (int)archive.ReadLong();
var storedDataSize = archive.ReadLong();

var v11Unknown1 = archive.ReadLong(); //file size or some other pointer
var v11Unknown2 = archive.ReadLong(); //0
var v11Unknown3 = archive.ReadLong(); //file size or some other pointer
var v11Unknown4 = archive.ReadLong(); //0
var v11Unknown5 = archive.ReadLong(); //file size or some other pointer
var v11Unknown6 = archive.ReadLong(); //0
}
else
{
storedOffset = 0;

//read in stored data file offset and sizes
for (int x = 0; x < 4; x++)
{
var storedOffset = new Tuple<long, long>(archive.ReadLong(), archive.ReadLong());
StoredDataOffsets.Add(storedOffset);
}
}

hibernationOffset = archive.ReadInt();
Expand Down Expand Up @@ -359,7 +359,6 @@ private void readBinaryHibernation(ArkArchive archive, ReadingOptions options)

archive.Position = hibernationOffset;

/*
if (SaveVersion > 7)
{
hibernationV8Unknown1 = archive.ReadInt();
Expand All @@ -375,24 +374,32 @@ private void readBinaryHibernation(ArkArchive archive, ReadingOptions options)
archive.DebugMessage("non-zero unknown V9 fields, expecting duplicated set per 349.10");
hibernationV9_34910Unknown1 = archive.ReadInt();
hibernationV9_34910Unknown2 = archive.ReadInt();
if (!(hibernationV8Unknown1 == archive.ReadInt()
&& hibernationV8Unknown2 == archive.ReadInt()
&& hibernationV8Unknown3 == archive.ReadInt()
&& hibernationV8Unknown4 == archive.ReadInt()
&& hibernationV9_34910Unknown1 == archive.ReadInt()
&& hibernationV9_34910Unknown2 == archive.ReadInt()))
{
throw new NotSupportedException("349.10 workaround for duplicate unknown hibernation bytes failed");
}
}
*/

while (archive.Position < nameTableOffset)
{
//regardless of anything before the hibernated class count always seems to follow immediately after this byte in all save formats I have worked on.
if (archive.ReadSByte() == -52)

hibernationV9Check1 = archive.ReadInt();
hibernationV9Check2 = archive.ReadInt();
hibernationV9Check3 = archive.ReadInt();
hibernationV9Check4 = archive.ReadInt();
hibernationV9Check5 = archive.ReadInt();
hibernationV9Check6 = archive.ReadInt();

if (!(hibernationV8Unknown1 == hibernationV9Check1
&& hibernationV8Unknown2 == hibernationV9Check2
&& hibernationV8Unknown3 == hibernationV9Check3
&& hibernationV8Unknown4 == hibernationV9Check4
&& hibernationV9_34910Unknown1 == hibernationV9Check5
&& hibernationV9_34910Unknown2 == hibernationV9Check6))
{
break;
if (SaveVersion > 10)
{
//TODO:// more data reads - is it always 112 bytes?
archive.SkipBytes(112);

}
else
{
throw new NotSupportedException("349.10 workaround for duplicate unknown hibernation bytes failed");
}
}
}

Expand All @@ -402,10 +409,9 @@ private void readBinaryHibernation(ArkArchive archive, ReadingOptions options)
return;
}

/*

hibernationUnknown1 = archive.ReadInt();
hibernationUnknown2 = archive.ReadInt();
*/

int hibernatedClassesCount = archive.ReadInt();

Expand Down Expand Up @@ -445,103 +451,149 @@ private void readBinaryStoredObjects(ArkArchive archive, ReadingOptions options)
{
if (!options.CryopodCreatures) return;

var inventoryContainers = Objects.Where(x => x.GetPropertyValue<ObjectReference>("MyInventoryComponent") != null).ToList();
var inventoryContainers = Objects.AsParallel().Where(x => x.GetPropertyValue<ObjectReference>("MyInventoryComponent") != null).ToList();

var validStored = Objects
var validStored = Objects.AsParallel()
.Where(o =>
(o.ClassName.Name.Contains("Cryopod") || o.ClassString.Contains("SoulTrap_") || o.ClassString.Contains("Vivarium_"))
&& o.HasAnyProperty("CustomItemDatas")
(o.ClassString.Contains("Cryopod") || o.ClassString.Contains("SoulTrap_") || o.ClassString.Contains("Vivarium_"))
&& o.GetTypedProperty<PropertyArray>("CustomItemDatas") != null
)
.ToArray();
.ToList();

foreach (var o in validStored)
List<CryoStoreData> storedOffsets = new List<CryoStoreData>();

validStored.ForEach(x =>
{
if (!(o.Properties.First(p => p.NameString == "CustomItemDatas") is PropertyArray customData)) continue;
if (!(x.Properties.First(p => p.NameString == "CustomItemDatas") is PropertyArray customData)) return;

long cryoDataOffset = 0;
int dataFile = 1;

if (
archive.SaveVersion > 10
&& customData.Value is ArkArrayStruct redirectors
&& redirectors.All(x => x is StructCustomItemDataRef)
)
&& redirectors.All(r => r is StructCustomItemDataRef)
)
{
// cryopods use the first redirector, soulball use the second
// cryopods use the first redirector, soulballs use the second
var redirectorIndex = o.ClassName.Name.Contains("Cryopod") ? 0 : 1;
cryoDataOffset = ((StructCustomItemDataRef)redirectors[redirectorIndex]).Position;
dataFile = ((StructCustomItemDataRef)redirectors[redirectorIndex]).StoreDataIndex;
}

var creatureDataOffset = cryoDataOffset + storedOffset;
archive.Position = creatureDataOffset;

ArkCryoStore cryoStore = new ArkCryoStore(archive);
if (!cryoStore.Objects.Any()) continue;
cryoStore.LoadProperties(archive);
storedOffsets.Add(new CryoStoreData() { ParentObject = x, Offset = cryoDataOffset, StoreDataIndex = dataFile });

var dinoComponent = cryoStore.Objects[0];
var dinoCharacterStatusComponent = cryoStore.Objects[1];
var dinoInventoryComponent = cryoStore.Objects[2];
});
validStored = null;

//re-map and add properties as appropriate
if (dinoComponent == null) continue;
//sort and read bytes into memory between this and next offset
var sortedOffsets = storedOffsets.OrderBy(x => x.StoreDataIndex).ThenBy(x => x.Offset).ToList();
for (int x = 0; x < sortedOffsets.Count; x++)
{
var storedObject = sortedOffsets[x];
int storedOffsetIndex = storedObject.StoreDataIndex;
var offsetData = StoredDataOffsets[storedOffsetIndex];

dinoComponent.IsInCryo = true;
var storedOffset = offsetData.Item1 + storedObject.Offset;
long nextOffset = offsetData.Item1 + offsetData.Item2;
if (x < sortedOffsets.Count - 1)
{
var nextItem = sortedOffsets[x + 1];
if (nextItem.StoreDataIndex == storedObject.StoreDataIndex)
{
nextOffset = nextItem.Offset + offsetData.Item1;
}
}

// the tribe name is stored in `TamerString`, non-cryoed creatures have the property `TribeName` for that.
if (dinoComponent.GetPropertyValue<string>("TribeName")?.Length == 0 && dinoComponent.GetPropertyValue<string>("TamerString")?.Length > 0)
dinoComponent.Properties.Add(new PropertyString("TribeName", dinoComponent.GetPropertyValue<string>("TamerString")));
archive.Position = storedOffset;
int bytesToRead = (int)(nextOffset - storedOffset);
storedObject.Data = archive.ReadBytes(bytesToRead);
}
storedOffsets = new List<CryoStoreData>();

//get parent of cryopod owner inventory
var podParentRef = o.GetPropertyValue<ObjectReference>("OwnerInventory");
if (podParentRef != null)
//load the cryo creature data in parallel to speed up read
Parallel.ForEach(sortedOffsets, o =>
{
if (o.Data.Length == 0) return;
using (MemoryStream ms = new MemoryStream(o.Data))
{
var podParent = inventoryContainers.FirstOrDefault(x => x.GetPropertyValue<ObjectReference>("MyInventoryComponent")?.ObjectId == podParentRef.ObjectId);

//determine if we need to re-team the podded animal
if (podParent != null)
using (ArkArchive storedArchive = new ArkArchive(ms))
{
dinoComponent.Parent = podParent;
ArkCryoStore cryoStore = new ArkCryoStore(storedArchive);
if (!cryoStore.Objects.Any()) return;

cryoStore.LoadProperties(storedArchive);


var dinoComponent = cryoStore.Objects[0];
var dinoCharacterStatusComponent = cryoStore.Objects[1];
var dinoInventoryComponent = cryoStore.Objects[2];

//re-map and add properties as appropriate
if (dinoComponent == null) return;

dinoComponent.IsInCryo = true;

int obTeam = dinoComponent.GetPropertyValue<int>("TargetingTeam");
int containerTeam = podParent.GetPropertyValue<int>("TargetingTeam");
if (obTeam != containerTeam)
// the tribe name is stored in `TamerString`, non-cryoed creatures have the property `TribeName` for that.
if (dinoComponent.GetPropertyValue<string>("TribeName")?.Length == 0 && dinoComponent.GetPropertyValue<string>("TamerString")?.Length > 0)
dinoComponent.Properties.Add(new PropertyString("TribeName", dinoComponent.GetPropertyValue<string>("TamerString")));

//get parent of cryopod owner inventory
var podParentRef = o.ParentObject.GetPropertyValue<ObjectReference>("OwnerInventory");
if (podParentRef != null)
{
var propertyIndex = dinoComponent.Properties.FindIndex(i => i.NameString == "TargetingTeam");
if (propertyIndex != -1)
var podParent = inventoryContainers.FirstOrDefault(x => x.GetPropertyValue<ObjectReference>("MyInventoryComponent")?.ObjectId == podParentRef.ObjectId);

//determine if we need to re-team the podded animal
if (podParent != null)
{
dinoComponent.Properties.RemoveAt(propertyIndex);
}
dinoComponent.Properties.Add(new PropertyInt("TargetingTeam", containerTeam));
dinoComponent.Parent = podParent;

int obTeam = dinoComponent.GetPropertyValue<int>("TargetingTeam");
int containerTeam = podParent.GetPropertyValue<int>("TargetingTeam");
if (obTeam != containerTeam)
{
var propertyIndex = dinoComponent.Properties.FindIndex(i => i.NameString == "TargetingTeam");
if (propertyIndex != -1)
{
dinoComponent.Properties.RemoveAt(propertyIndex);
}
dinoComponent.Properties.Add(new PropertyInt("TargetingTeam", containerTeam));

if (dinoComponent.HasAnyProperty("TamingTeamID"))
{
dinoComponent.Properties.RemoveAt(dinoComponent.Properties.FindIndex(i => i.NameString == "TamingTeamID"));
dinoComponent.Properties.Add(new PropertyInt("TamingTeamID", containerTeam));
}

if (dinoComponent.HasAnyProperty("TamingTeamID"))
{
dinoComponent.Properties.RemoveAt(dinoComponent.Properties.FindIndex(i => i.NameString == "TamingTeamID"));
dinoComponent.Properties.Add(new PropertyInt("TamingTeamID", containerTeam));
}

}
}
}
}
}

if (dinoCharacterStatusComponent != null)
{
addObject(dinoCharacterStatusComponent, true);
if (dinoCharacterStatusComponent != null)
{
addObject(dinoCharacterStatusComponent, false);

var statusComponentRef = dinoComponent.GetTypedProperty<PropertyObject>("MyCharacterStatusComponent");
statusComponentRef.Value.ObjectId = dinoCharacterStatusComponent.Id;
var statusComponentRef = dinoComponent.GetTypedProperty<PropertyObject>("MyCharacterStatusComponent");
statusComponentRef.Value.ObjectId = dinoCharacterStatusComponent.Id;

}
}

if (dinoInventoryComponent != null)
{
addObject(dinoInventoryComponent, true);
if (dinoInventoryComponent != null)
{
addObject(dinoInventoryComponent, false);

var inventoryComponentRef = dinoComponent.GetTypedProperty<PropertyObject>("MyInventoryComponent");
inventoryComponentRef.Value.ObjectId = dinoInventoryComponent.Id;
}

var inventoryComponentRef = dinoComponent.GetTypedProperty<PropertyObject>("MyInventoryComponent");
inventoryComponentRef.Value.ObjectId = dinoInventoryComponent.Id;
addObject(dinoComponent, false);

}
}
});

addObject(dinoComponent, true);
}
}


Expand Down
11 changes: 7 additions & 4 deletions SavegameToolkit/Structs/StructCustomItemDataRef.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,21 @@ namespace SavegameToolkit.Structs {
public class StructCustomItemDataRef : StructBase {

[JsonProperty(Order = 0)]
public short Unknown0 { get; private set; }
public byte Unknown0 { get; private set; }
[JsonProperty(Order = 1)]
public long Position { get; private set; }
public byte StoreDataIndex{ get; private set; }
[JsonProperty(Order = 2)]
public ObjectReference[] ObjectRefs { get; private set; }
public long Position { get; private set; }
[JsonProperty(Order = 3)]
public ObjectReference[] ObjectRefs { get; private set; }
[JsonProperty(Order = 4)]
public ObjectReference[] ClassRefs { get; private set; }

public override void Init(ArkArchive archive)
{
// The first unknown field may be two fields - perhaps format version and archive index
Unknown0 = archive.ReadShort();
Unknown0 = archive.ReadByte();
StoreDataIndex = archive.ReadByte();
Position = archive.ReadLong();
ObjectRefs = new ObjectReference[archive.ReadInt()];
for (int index = 0; index < ObjectRefs.Length; index++)
Expand Down
16 changes: 16 additions & 0 deletions SavegameToolkit/Types/CryoStoreData.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;

namespace SavegameToolkit.Types
{
public class CryoStoreData
{

public GameObject ParentObject { get; set; }
public int StoreDataIndex { get; set; } = 1;
public long Offset { get; set; }
public byte[] Data { get; set; }

}
}

0 comments on commit 7e0b4a2

Please sign in to comment.