Protobuf based query builders

TrinityCore · Jan 12, 2025 · 852fea1 · 852fea1
1 parent 49a2a48
commit 852fea1
Show file tree

Hide file tree

Showing 7 changed files with 138 additions and 22 deletions.
diff --git a/WowPacketParser/Loading/SniffFile.cs b/WowPacketParser/Loading/SniffFile.cs
@@ -83,18 +83,19 @@ private string FileName
             }
         }
 
-        public void ProcessFile()
+        public Packets ProcessFile()
         {
 
             try
             {
-                ProcessFileImpl();
+                return ProcessFileImpl();
             }
             catch (Exception ex)
             {
                 Trace.WriteLine(_logPrefix + " " + ex.GetType());
                 Trace.WriteLine(_logPrefix + " " + ex.Message);
                 Trace.WriteLine(_logPrefix + " " + ex.StackTrace);
+                return new Packets();
             }
             finally
             {
@@ -106,7 +107,7 @@ public void ProcessFile()
             }
         }
 
-        private void ProcessFileImpl()
+        private Packets ProcessFileImpl()
         {
             if (_compression != FileCompression.None)
                 _tempName = Decompress();
@@ -184,9 +185,9 @@ private void ProcessFileImpl()
 
                     var written = false;
 
+                    Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat };
                     using (var writer = (Settings.DumpFormatWithTextToFile() ? new StreamWriter(outFileName, true) : null))
                     {
-                        Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat };
                         var firstRead = true;
                         var firstWrite = true;
 
@@ -285,7 +286,7 @@ private void ProcessFileImpl()
                             // Close Writer, Stream - Dispose
                             packet.ClosePacket();
 
-                            if (_dumpFormat.IsUniversalProtobufType())
+                            if (_dumpFormat.IsUniversalProtobufType() || Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
                                 packets.Packets_.Add(packet.Holder);
                         }, threadCount);
 
@@ -316,14 +317,14 @@ private void ProcessFileImpl()
                     Trace.WriteLine($"{_logPrefix}: {_stats}");
 
                     if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
-                        WriteSQLs();
+                        WriteSQLs(packets);
 
                     if (Settings.LogPacketErrors)
                         WritePacketErrors();
 
                     GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
 
-                    break;
+                    return packets;
                 }
                 case DumpFormatType.Pkt:
                 {
@@ -478,6 +479,8 @@ private void ProcessFileImpl()
                     break;
                 }
             }
+
+            return new Packets();
         }
 
         public static string GetHeader(string fileName)
@@ -560,14 +563,14 @@ private void BinaryDump(string fileName, ICollection<Packet> packets)
             BinaryPacketWriter.Write(SniffType.Pkt, fileName, Encoding.ASCII, packets);
         }
 
-        private void WriteSQLs()
+        private void WriteSQLs(Packets packets)
         {
             var sqlFileName = string.IsNullOrWhiteSpace(Settings.SQLFileName) ? $"{Utilities.FormattedDateTimeForFiles()}_{Path.GetFileName(FileName)}.sql" : Settings.SQLFileName;
 
             if (!string.IsNullOrWhiteSpace(Settings.SQLFileName))
                 return;
 
-            Builder.DumpSQL($"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName));
+            Builder.DumpSQL(new []{packets}, $"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName));
             Storage.ClearContainers();
         }
 

diff --git a/WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs b/WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs
@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+using WowPacketParser.Proto;
+using WowPacketParser.Proto.Processing;
+
+namespace WowPacketParser.Parsing.Proto;
+
+public abstract class BaseProtoQueryBuilder : PacketProcessor<VoidType>, IProtoQueryBuilder
+{
+    public abstract bool IsEnabled();
+
+    public string Process(IReadOnlyList<Packets> packetsList)
+    {
+        foreach (var sniffFile in packetsList)
+        {
+            // more complex logic is possible, i.e. clear state after each file
+            foreach (var packet in sniffFile.Packets_)
+            {
+                Process(packet);
+            }
+        }
+
+        return GenerateQuery();
+    }
+
+    protected abstract string GenerateQuery();
+}
diff --git a/WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs b/WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs
@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using WowPacketParser.Proto;
+
+namespace WowPacketParser.Parsing.Proto;
+
+public interface IProtoQueryBuilder
+{
+    bool IsEnabled();
+    string Process(IReadOnlyList<Packets> packets);
+}
diff --git a/WowPacketParser/Parsing/Proto/VoidType.cs b/WowPacketParser/Parsing/Proto/VoidType.cs
@@ -0,0 +1,3 @@
+namespace WowPacketParser.Parsing.Proto;
+
+public struct VoidType { }
diff --git a/WowPacketParser/Program.cs b/WowPacketParser/Program.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -9,6 +10,7 @@
 using WowPacketParser.Loading;
 using WowPacketParser.Misc;
 using WowPacketParser.Parsing.Parsers;
+using WowPacketParser.Proto;
 using WowPacketParser.SQL;
 
 namespace WowPacketParser
@@ -77,6 +79,8 @@ private static void Main(string[] args)
 
             SQLConnector.ReadDB();
 
+            List<Packets> parserPacketsList = new();
+
             var processStartTime = DateTime.Now;
             var count = 0;
             foreach (var file in files)
@@ -90,7 +94,7 @@ private static void Main(string[] args)
                 try
                 {
                     var sf = new SniffFile(file, Settings.DumpFormat, Tuple.Create(++count, files.Count));
-                    sf.ProcessFile();
+                    parserPacketsList.Add(sf.ProcessFile());
                 }
                 catch (IOException ex)
                 {
@@ -99,7 +103,7 @@ private static void Main(string[] args)
             }
 
             if (!string.IsNullOrWhiteSpace(Settings.SQLFileName) && Settings.DumpFormatWithSQL())
-                Builder.DumpSQL("Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi"));
+                Builder.DumpSQL(parserPacketsList, "Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi"));
 
             var processTime = DateTime.Now.Subtract(processStartTime);
             Trace.WriteLine($"Processing {files.Count} sniffs took { processTime.ToFormattedString() }.");

diff --git a/WowPacketParser/SQL/Builder.cs b/WowPacketParser/SQL/Builder.cs
@@ -5,6 +5,8 @@
 using System.Reflection;
 using WowPacketParser.Enums;
 using WowPacketParser.Misc;
+using WowPacketParser.Parsing.Proto;
+using WowPacketParser.Proto;
 using WowPacketParser.SQL.Builders;
 using WowPacketParser.Store;
 using WowPacketParser.Store.Objects;
@@ -63,24 +65,47 @@ private static void LoadNames()
             }
         }
 
-        public static void DumpFile(string prefix, string fileName, string header, List<MethodInfo> builderMethods, Dictionary<WowGuid, Unit> units, Dictionary<WowGuid, GameObject> gameObjects)
+        public static void DumpFile(IReadOnlyList<Packets> packets, string prefix, string fileName, string header, List<Type> protoBuilders, List<MethodInfo> builderMethods, Dictionary<WowGuid, Unit> units, Dictionary<WowGuid, GameObject> gameObjects)
         {
             var startTime = DateTime.Now;
             using (var store = new SQLFile(fileName))
             {
                 store.WriteData(UnitMisc.CreatureEquip(units)); // ensure this is run before spawns
 
-                for (int i = 1; i <= builderMethods.Count; i++)
+                var currentBuilder = 0;
+                var totalCount = protoBuilders.Count + builderMethods.Count;
+
+                foreach (var builderType in protoBuilders)
+                {
+                    currentBuilder++;
+                    var builder = (IProtoQueryBuilder)Activator.CreateInstance(builderType);
+
+                    if (!builder.IsEnabled())
+                        continue;
+
+                    Trace.WriteLine($"{currentBuilder}/{totalCount} - Write {builderType}");
+                    try
+                    {
+                        store.WriteData(builder.Process(packets));
+                    }
+                    catch (TargetInvocationException e)
+                    {
+                        Trace.WriteLine($"{currentBuilder}/{totalCount} - Error: Failed writing {builderType}");
+                        Trace.TraceError(e.InnerException?.ToString() ?? e.ToString());
+                    }
+                }
+
+                foreach (var method in builderMethods)
                 {
-                    var method = builderMethods[i - 1];
+                    currentBuilder++;
                     var attr = method.GetCustomAttribute<BuilderMethodAttribute>();
 
                     if (attr.CheckVersionMismatch)
                     {
                         if (!GetExpectedTargetDatabasesForExpansion(ClientVersion.Expansion).Contains(Settings.TargetedDatabase))
                         {
                             Trace.WriteLine(
-                                $"{i}/{builderMethods.Count} - Error: Couldn't generate SQL output of {method.Name} since the targeted database and the sniff version don't match.");
+                                $"{currentBuilder}/{totalCount} - Error: Couldn't generate SQL output of {method.Name} since the targeted database and the sniff version don't match.");
                             continue;
                         }
                     }
@@ -92,14 +117,14 @@ public static void DumpFile(string prefix, string fileName, string header, List<
                     if (attr.Gameobjects)
                         parameters.Add(gameObjects);
 
-                    Trace.WriteLine($"{i}/{builderMethods.Count} - Write {method.Name}");
+                    Trace.WriteLine($"{currentBuilder}/{totalCount} - Write {method.Name}");
                     try
                     {
                         store.WriteData(method.Invoke(null, parameters.ToArray()).ToString());
                     }
                     catch (TargetInvocationException e)
                     {
-                        Trace.WriteLine($"{i}/{builderMethods.Count} - Error: Failed writing {method.Name}");
+                        Trace.WriteLine($"{currentBuilder}/{totalCount} - Error: Failed writing {method.Name}");
                         Trace.TraceError(e.InnerException?.ToString() ?? e.ToString());
                     }
                 }
@@ -113,7 +138,7 @@ public static void DumpFile(string prefix, string fileName, string header, List<
             }
         }
 
-        public static void DumpSQL(string prefix, string fileName, string header)
+        public static void DumpSQL(IReadOnlyList<Packets> packets, string prefix, string fileName, string header)
         {
             var startTime = DateTime.Now;
 
@@ -140,23 +165,33 @@ public static void DumpSQL(string prefix, string fileName, string header)
                     .SelectMany(x => x.GetMethods());
             var allMethods = methods.Select(y => new { Method = y, Attributes = y.GetCustomAttributes().OfType<BuilderMethodAttribute>()}).Where(y => y.Attributes.Any()).ToList();
 
+            var allProtoBuilders = Assembly.GetExecutingAssembly()
+                .GetTypes()
+                .Select(type => (Type: type, Attributes: type.GetCustomAttributes(typeof(ProtoBuilderClassAttribute), true).OfType<ProtoBuilderClassAttribute>().ToList()))
+                .Where(pair => pair.Attributes.Count > 0)
+                .ToList();
+
             if (Settings.SplitSQLFile)
             {
                 fileName = System.IO.Path.ChangeExtension(fileName, null); // remove .sql
 
                 var hotfixMethods = allMethods.Where(x => x.Attributes.First().Database == TargetSQLDatabase.Hotfixes).Select(x => x.Method).ToList();
-                DumpFile(prefix, $"{fileName}_hotfixes.sql", header, hotfixMethods, units, gameObjects);
+                var hotfixProtoBuilders = allProtoBuilders.Where(x => x.Attributes.First().Database == TargetSQLDatabase.Hotfixes).Select(x => x.Type).ToList();
+                DumpFile(packets, prefix, $"{fileName}_hotfixes.sql", header, hotfixProtoBuilders, hotfixMethods, units, gameObjects);
 
                 var worldMethods = allMethods.Where(x => x.Attributes.First().Database == TargetSQLDatabase.World).Select(x => x.Method).ToList();
-                DumpFile(prefix, $"{fileName}_world.sql", header, worldMethods, units, gameObjects);
+                var worldProtoBuilders = allProtoBuilders.Where(x => x.Attributes.First().Database == TargetSQLDatabase.World).Select(x => x.Type).ToList();
+                DumpFile(packets, prefix, $"{fileName}_world.sql", header, worldProtoBuilders, worldMethods, units, gameObjects);
 
                 var wppMethods = allMethods.Where(x => x.Attributes.First().Database == TargetSQLDatabase.WPP).Select(x => x.Method).ToList();
-                DumpFile(prefix, $"{fileName}_wpp.sql", header, wppMethods, units, gameObjects);
+                var wppProtoBuilders = allProtoBuilders.Where(x => x.Attributes.First().Database == TargetSQLDatabase.WPP).Select(x => x.Type).ToList();
+                DumpFile(packets, prefix, $"{fileName}_wpp.sql", header, wppProtoBuilders, wppMethods, units, gameObjects);
             }
             else
             {
+                var protoBuilderTypes = allProtoBuilders.Select(x => x.Type).ToList();
                 var builderMethods = allMethods.Select(x => x.Method).ToList();
-                DumpFile(prefix, fileName, header, builderMethods, units, gameObjects);
+                DumpFile(packets, prefix, fileName, header, protoBuilderTypes, builderMethods, units, gameObjects);
             }
         }
 

diff --git a/WowPacketParser/SQL/Builders/BuilderAttributes.cs b/WowPacketParser/SQL/Builders/BuilderAttributes.cs
@@ -55,4 +55,39 @@ public BuilderMethodAttribute(bool checkVersionMissmatch, TargetSQLDatabase data
     public sealed class BuilderClassAttribute : Attribute
     {
     }
+
+    [AttributeUsage(AttributeTargets.Class)]
+    public sealed class ProtoBuilderClassAttribute : Attribute
+    {
+        public ProtoBuilderClassAttribute()
+        {
+            Database = TargetSQLDatabase.World;
+        }
+        public ProtoBuilderClassAttribute(TargetSQLDatabase database)
+        {
+            Database = database;
+        }
+
+        public ProtoBuilderClassAttribute(bool checkVersionMissmatch)
+        {
+            Database = TargetSQLDatabase.World;
+            CheckVersionMismatch = checkVersionMissmatch;
+        }
+
+        public ProtoBuilderClassAttribute(bool checkVersionMissmatch, TargetSQLDatabase database)
+        {
+            Database = database;
+            CheckVersionMismatch = checkVersionMissmatch;
+        }
+
+        /// <summary>
+        /// True if mismatch between targeted database and sniff version should be checked
+        /// </summary>
+        public bool CheckVersionMismatch { get; private set; }
+
+        // <summary>
+        // Defines the targeted database
+        // </summary>
+        public TargetSQLDatabase Database;
+    }
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		namespace WowPacketParser.Parsing.Proto;

		public struct VoidType { }