diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs index fe23ffffc9..ed1278b410 100644 --- a/WowPacketParser/Loading/SniffFile.cs +++ b/WowPacketParser/Loading/SniffFile.cs @@ -83,18 +83,19 @@ private string FileName } } - public void ProcessFile() + public Packets ProcessFile() { try { - ProcessFileImpl(); + return ProcessFileImpl(); } catch (Exception ex) { Trace.WriteLine(_logPrefix + " " + ex.GetType()); Trace.WriteLine(_logPrefix + " " + ex.Message); Trace.WriteLine(_logPrefix + " " + ex.StackTrace); + return new Packets(); } finally { @@ -106,7 +107,7 @@ public void ProcessFile() } } - private void ProcessFileImpl() + private Packets ProcessFileImpl() { if (_compression != FileCompression.None) _tempName = Decompress(); @@ -184,9 +185,9 @@ private void ProcessFileImpl() var written = false; + Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat }; using (var writer = (Settings.DumpFormatWithTextToFile() ? new StreamWriter(outFileName, true) : null)) { - Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat }; var firstRead = true; var firstWrite = true; @@ -285,7 +286,7 @@ private void ProcessFileImpl() // Close Writer, Stream - Dispose packet.ClosePacket(); - if (_dumpFormat.IsUniversalProtobufType()) + if (_dumpFormat.IsUniversalProtobufType() || Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog()) packets.Packets_.Add(packet.Holder); }, threadCount); @@ -316,14 +317,14 @@ private void ProcessFileImpl() Trace.WriteLine($"{_logPrefix}: {_stats}"); if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog()) - WriteSQLs(); + WriteSQLs(packets); if (Settings.LogPacketErrors) WritePacketErrors(); GC.Collect(); // Force a GC collect after parsing a file. It seems to help. - break; + return packets; } case DumpFormatType.Pkt: { @@ -478,6 +479,8 @@ private void ProcessFileImpl() break; } } + + return new Packets(); } public static string GetHeader(string fileName) @@ -560,14 +563,14 @@ private void BinaryDump(string fileName, ICollection packets) BinaryPacketWriter.Write(SniffType.Pkt, fileName, Encoding.ASCII, packets); } - private void WriteSQLs() + private void WriteSQLs(Packets packets) { var sqlFileName = string.IsNullOrWhiteSpace(Settings.SQLFileName) ? $"{Utilities.FormattedDateTimeForFiles()}_{Path.GetFileName(FileName)}.sql" : Settings.SQLFileName; if (!string.IsNullOrWhiteSpace(Settings.SQLFileName)) return; - Builder.DumpSQL($"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName)); + Builder.DumpSQL(new []{packets}, $"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName)); Storage.ClearContainers(); } diff --git a/WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs b/WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs new file mode 100644 index 0000000000..30c4b8caec --- /dev/null +++ b/WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using WowPacketParser.Proto; +using WowPacketParser.Proto.Processing; + +namespace WowPacketParser.Parsing.Proto; + +public abstract class BaseProtoQueryBuilder : PacketProcessor, IProtoQueryBuilder +{ + public abstract bool IsEnabled(); + + public string Process(IReadOnlyList packetsList) + { + foreach (var sniffFile in packetsList) + { + // more complex logic is possible, i.e. clear state after each file + foreach (var packet in sniffFile.Packets_) + { + Process(packet); + } + } + + return GenerateQuery(); + } + + protected abstract string GenerateQuery(); +} \ No newline at end of file diff --git a/WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs b/WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs new file mode 100644 index 0000000000..e9b08dc6aa --- /dev/null +++ b/WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using WowPacketParser.Proto; + +namespace WowPacketParser.Parsing.Proto; + +public interface IProtoQueryBuilder +{ + bool IsEnabled(); + string Process(IReadOnlyList packets); +} \ No newline at end of file diff --git a/WowPacketParser/Parsing/Proto/VoidType.cs b/WowPacketParser/Parsing/Proto/VoidType.cs new file mode 100644 index 0000000000..7d0e7f8203 --- /dev/null +++ b/WowPacketParser/Parsing/Proto/VoidType.cs @@ -0,0 +1,3 @@ +namespace WowPacketParser.Parsing.Proto; + +public struct VoidType { } \ No newline at end of file diff --git a/WowPacketParser/Program.cs b/WowPacketParser/Program.cs index c147a34603..8f5c339d8b 100644 --- a/WowPacketParser/Program.cs +++ b/WowPacketParser/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; @@ -9,6 +10,7 @@ using WowPacketParser.Loading; using WowPacketParser.Misc; using WowPacketParser.Parsing.Parsers; +using WowPacketParser.Proto; using WowPacketParser.SQL; namespace WowPacketParser @@ -77,6 +79,8 @@ private static void Main(string[] args) SQLConnector.ReadDB(); + List parserPacketsList = new(); + var processStartTime = DateTime.Now; var count = 0; foreach (var file in files) @@ -90,7 +94,7 @@ private static void Main(string[] args) try { var sf = new SniffFile(file, Settings.DumpFormat, Tuple.Create(++count, files.Count)); - sf.ProcessFile(); + parserPacketsList.Add(sf.ProcessFile()); } catch (IOException ex) { @@ -99,7 +103,7 @@ private static void Main(string[] args) } if (!string.IsNullOrWhiteSpace(Settings.SQLFileName) && Settings.DumpFormatWithSQL()) - Builder.DumpSQL("Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi")); + Builder.DumpSQL(parserPacketsList, "Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi")); var processTime = DateTime.Now.Subtract(processStartTime); Trace.WriteLine($"Processing {files.Count} sniffs took { processTime.ToFormattedString() }."); diff --git a/WowPacketParser/SQL/Builder.cs b/WowPacketParser/SQL/Builder.cs index bf82a11d4f..ef3c574a5b 100644 --- a/WowPacketParser/SQL/Builder.cs +++ b/WowPacketParser/SQL/Builder.cs @@ -5,6 +5,8 @@ using System.Reflection; using WowPacketParser.Enums; using WowPacketParser.Misc; +using WowPacketParser.Parsing.Proto; +using WowPacketParser.Proto; using WowPacketParser.SQL.Builders; using WowPacketParser.Store; using WowPacketParser.Store.Objects; @@ -63,16 +65,39 @@ private static void LoadNames() } } - public static void DumpFile(string prefix, string fileName, string header, List builderMethods, Dictionary units, Dictionary gameObjects) + public static void DumpFile(IReadOnlyList packets, string prefix, string fileName, string header, List protoBuilders, List builderMethods, Dictionary units, Dictionary gameObjects) { var startTime = DateTime.Now; using (var store = new SQLFile(fileName)) { store.WriteData(UnitMisc.CreatureEquip(units)); // ensure this is run before spawns - for (int i = 1; i <= builderMethods.Count; i++) + var currentBuilder = 0; + var totalCount = protoBuilders.Count + builderMethods.Count; + + foreach (var builderType in protoBuilders) + { + currentBuilder++; + var builder = (IProtoQueryBuilder)Activator.CreateInstance(builderType); + + if (!builder.IsEnabled()) + continue; + + Trace.WriteLine($"{currentBuilder}/{totalCount} - Write {builderType}"); + try + { + store.WriteData(builder.Process(packets)); + } + catch (TargetInvocationException e) + { + Trace.WriteLine($"{currentBuilder}/{totalCount} - Error: Failed writing {builderType}"); + Trace.TraceError(e.InnerException?.ToString() ?? e.ToString()); + } + } + + foreach (var method in builderMethods) { - var method = builderMethods[i - 1]; + currentBuilder++; var attr = method.GetCustomAttribute(); if (attr.CheckVersionMismatch) @@ -80,7 +105,7 @@ public static void DumpFile(string prefix, string fileName, string header, List< if (!GetExpectedTargetDatabasesForExpansion(ClientVersion.Expansion).Contains(Settings.TargetedDatabase)) { Trace.WriteLine( - $"{i}/{builderMethods.Count} - Error: Couldn't generate SQL output of {method.Name} since the targeted database and the sniff version don't match."); + $"{currentBuilder}/{totalCount} - Error: Couldn't generate SQL output of {method.Name} since the targeted database and the sniff version don't match."); continue; } } @@ -92,14 +117,14 @@ public static void DumpFile(string prefix, string fileName, string header, List< if (attr.Gameobjects) parameters.Add(gameObjects); - Trace.WriteLine($"{i}/{builderMethods.Count} - Write {method.Name}"); + Trace.WriteLine($"{currentBuilder}/{totalCount} - Write {method.Name}"); try { store.WriteData(method.Invoke(null, parameters.ToArray()).ToString()); } catch (TargetInvocationException e) { - Trace.WriteLine($"{i}/{builderMethods.Count} - Error: Failed writing {method.Name}"); + Trace.WriteLine($"{currentBuilder}/{totalCount} - Error: Failed writing {method.Name}"); Trace.TraceError(e.InnerException?.ToString() ?? e.ToString()); } } @@ -113,7 +138,7 @@ public static void DumpFile(string prefix, string fileName, string header, List< } } - public static void DumpSQL(string prefix, string fileName, string header) + public static void DumpSQL(IReadOnlyList packets, string prefix, string fileName, string header) { var startTime = DateTime.Now; @@ -140,23 +165,33 @@ public static void DumpSQL(string prefix, string fileName, string header) .SelectMany(x => x.GetMethods()); var allMethods = methods.Select(y => new { Method = y, Attributes = y.GetCustomAttributes().OfType()}).Where(y => y.Attributes.Any()).ToList(); + var allProtoBuilders = Assembly.GetExecutingAssembly() + .GetTypes() + .Select(type => (Type: type, Attributes: type.GetCustomAttributes(typeof(ProtoBuilderClassAttribute), true).OfType().ToList())) + .Where(pair => pair.Attributes.Count > 0) + .ToList(); + if (Settings.SplitSQLFile) { fileName = System.IO.Path.ChangeExtension(fileName, null); // remove .sql var hotfixMethods = allMethods.Where(x => x.Attributes.First().Database == TargetSQLDatabase.Hotfixes).Select(x => x.Method).ToList(); - DumpFile(prefix, $"{fileName}_hotfixes.sql", header, hotfixMethods, units, gameObjects); + var hotfixProtoBuilders = allProtoBuilders.Where(x => x.Attributes.First().Database == TargetSQLDatabase.Hotfixes).Select(x => x.Type).ToList(); + DumpFile(packets, prefix, $"{fileName}_hotfixes.sql", header, hotfixProtoBuilders, hotfixMethods, units, gameObjects); var worldMethods = allMethods.Where(x => x.Attributes.First().Database == TargetSQLDatabase.World).Select(x => x.Method).ToList(); - DumpFile(prefix, $"{fileName}_world.sql", header, worldMethods, units, gameObjects); + var worldProtoBuilders = allProtoBuilders.Where(x => x.Attributes.First().Database == TargetSQLDatabase.World).Select(x => x.Type).ToList(); + DumpFile(packets, prefix, $"{fileName}_world.sql", header, worldProtoBuilders, worldMethods, units, gameObjects); var wppMethods = allMethods.Where(x => x.Attributes.First().Database == TargetSQLDatabase.WPP).Select(x => x.Method).ToList(); - DumpFile(prefix, $"{fileName}_wpp.sql", header, wppMethods, units, gameObjects); + var wppProtoBuilders = allProtoBuilders.Where(x => x.Attributes.First().Database == TargetSQLDatabase.WPP).Select(x => x.Type).ToList(); + DumpFile(packets, prefix, $"{fileName}_wpp.sql", header, wppProtoBuilders, wppMethods, units, gameObjects); } else { + var protoBuilderTypes = allProtoBuilders.Select(x => x.Type).ToList(); var builderMethods = allMethods.Select(x => x.Method).ToList(); - DumpFile(prefix, fileName, header, builderMethods, units, gameObjects); + DumpFile(packets, prefix, fileName, header, protoBuilderTypes, builderMethods, units, gameObjects); } } diff --git a/WowPacketParser/SQL/Builders/BuilderAttributes.cs b/WowPacketParser/SQL/Builders/BuilderAttributes.cs index 68c2f5d2dd..9acb063770 100644 --- a/WowPacketParser/SQL/Builders/BuilderAttributes.cs +++ b/WowPacketParser/SQL/Builders/BuilderAttributes.cs @@ -55,4 +55,39 @@ public BuilderMethodAttribute(bool checkVersionMissmatch, TargetSQLDatabase data public sealed class BuilderClassAttribute : Attribute { } + + [AttributeUsage(AttributeTargets.Class)] + public sealed class ProtoBuilderClassAttribute : Attribute + { + public ProtoBuilderClassAttribute() + { + Database = TargetSQLDatabase.World; + } + public ProtoBuilderClassAttribute(TargetSQLDatabase database) + { + Database = database; + } + + public ProtoBuilderClassAttribute(bool checkVersionMissmatch) + { + Database = TargetSQLDatabase.World; + CheckVersionMismatch = checkVersionMissmatch; + } + + public ProtoBuilderClassAttribute(bool checkVersionMissmatch, TargetSQLDatabase database) + { + Database = database; + CheckVersionMismatch = checkVersionMissmatch; + } + + /// + /// True if mismatch between targeted database and sniff version should be checked + /// + public bool CheckVersionMismatch { get; private set; } + + // + // Defines the targeted database + // + public TargetSQLDatabase Database; + } }