Merge pull request #55 from NeilMacMullen/misc-changes

Misc changes
NeilMacMullen · Aug 12, 2024 · e003477 · e003477
2 parents 73a6f6c + e15480f
commit e003477
Show file tree

Hide file tree

Showing 20 changed files with 204 additions and 40 deletions.
diff --git a/libraries/FileFormats/CsvSerializer.cs b/libraries/FileFormats/CsvSerializer.cs
@@ -1,4 +1,5 @@
 using System.Globalization;
+using System.Runtime.InteropServices;
 using System.Text;
 using CsvHelper;
 using CsvHelper.Configuration;
@@ -23,7 +24,9 @@ public CsvSerializer(CsvConfiguration config, KustoSettingsProvider settings, IK
         _console = console;
         settings.Register(CsvSerializerSettings.SkipTypeInference,
             CsvSerializerSettings.TrimCells,
-            CsvSerializerSettings.SkipHeaderOnSave);
+            CsvSerializerSettings.SkipHeaderOnSave,
+            CsvSerializerSettings.InferColumnNames,
+            CsvSerializerSettings.Separator);
     }
 
     public async Task<TableSaveResult> SaveTable(string path, KustoQueryResult result)
@@ -35,28 +38,48 @@ public async Task<TableSaveResult> SaveTable(string path, KustoQueryResult resul
 
     public Task<TableLoadResult> LoadTable(Stream stream, string tableName)
     {
+        var keys = Array.Empty<string>();
+        var builders = Array.Empty<ColumnBuilder<string>>();
+        var inferColumnNames = _settings.GetBool(CsvSerializerSettings.InferColumnNames);
         using var reader = new StreamReader(stream);
-        var csv = new CsvReader(reader, _config);
-        csv.Read();
-        csv.ReadHeader();
-
-        var keys = csv.Context.Reader?.HeaderRecord;
-
+        var config = _config with { HasHeaderRecord = !inferColumnNames };
+        var separator = (_settings.Get(CsvSerializerSettings.Separator));
+        if (separator.Length>=1)
+        {
+            config= config with { Delimiter = separator};
+        }
+
+        var csv = new CsvReader(reader, config);
+
+        if (!inferColumnNames)
+        {
+            csv.Read();
+            csv.ReadHeader();
+            keys = csv.Context.Reader?.HeaderRecord;
+            builders= keys
+                .Select(_ => new ColumnBuilder<string>())
+                .ToArray();
+        }
 
-        var builders = keys
-            .Select(_ => new ColumnBuilder<string>())
-            .ToArray();
         var rowCount = 0;
         while (csv.Read())
         {
+            if (!builders.Any())
+            {
+                keys = Enumerable.Range(0, csv.ColumnCount).Select(c => $"Column{c}").ToArray();
+                builders = keys
+                    .Select(_ => new ColumnBuilder<string>())
+                    .ToArray();
+            }
+
             var isTrimRequired = _settings.GetBool(CsvSerializerSettings.TrimCells);
 
             string TrimIfRequired(string s)
             {
                 return isTrimRequired ? s.Trim() : s;
             }
-
-            for (var i = 0; i < keys.Length; i++) builders[i].Add(TrimIfRequired(csv.GetField<string>(i)));
+           
+            for (var i = 0; i < builders.Length; i++) builders[i].Add(TrimIfRequired(csv.GetField<string>(i)));
 
             rowCount++;
             if (rowCount % 100_000 == 0)
@@ -160,7 +183,7 @@ private static class CsvSerializerSettings
         private const string Prefix = "csv";
 
         public static readonly KustoSettingDefinition SkipTypeInference = new(
-            Setting("skipTypeInference"), "prevents conversion of string columns to types",
+            Setting("SkipTypeInference"), "prevents conversion of string columns to types",
             "off",
             nameof(Boolean));
 
@@ -171,6 +194,14 @@ private static class CsvSerializerSettings
         public static readonly KustoSettingDefinition SkipHeaderOnSave = new(Setting("SkipHeaderOnSave"),
             "Don't write header row when saving CSV files", "false", nameof(Boolean));
 
+
+
+        public static readonly KustoSettingDefinition InferColumnNames = new(Setting("InferColumnNames"),
+            "Infer column names", "false", nameof(Boolean));
+
+        public static readonly KustoSettingDefinition Separator = new(Setting("Separator"),
+            "Character that separates columns", string.Empty, nameof(String));
+
         private static string Setting(string setting)
         {
             return $"{Prefix}.{setting}";

diff --git a/libraries/KustoLoco.Core/ColumnTypeInferrer.cs b/libraries/KustoLoco.Core/ColumnTypeInferrer.cs
@@ -21,11 +21,10 @@ public static class ColumnTypeInferrer
     private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
     private static readonly TypeTrier [] TypeTriers =
     [
-
         //don't bother with int .... Kusto is natively "long"
         //so this would just lead to excessive casts
         new TypeTrier(typeof(long), s => (long.TryParse(s, CultureInfo.InvariantCulture, out var i), i)),
-        new TypeTrier(typeof(double), s => (double.TryParse(s, CultureInfo.InvariantCulture, out var i), i)),
+        new TypeTrier(typeof(double), s => s.Length <=17 && double.TryParse(s, CultureInfo.InvariantCulture, out var i) ? (true,i) :(false,0)),
         new TypeTrier(typeof(DateTime), s => (DateTime.TryParse(s, CultureInfo.InvariantCulture, out var i), i)),
         new TypeTrier(typeof(Guid), s => (Guid.TryParse(s, CultureInfo.InvariantCulture, out var i), i)),
         new TypeTrier(typeof(TimeSpan), s => (TimeSpan.TryParse(s, CultureInfo.InvariantCulture, out var i), i)),
@@ -76,4 +75,4 @@ public static BaseColumn AutoInfer(BaseColumn source)
     }
 
     private readonly record struct TypeTrier(Type Type, Func<string, (bool, object)> Parser);
-}
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/AvgAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/AvgAggregate.cs
@@ -60,5 +60,5 @@ internal static DateTime DtImpl(NumericAggregate context, DateTime n)
 
     internal static DateTime? DtImplFinish(NumericAggregate context) => context.Count == 0
         ? null
-        : new DateTime((long)context.Total / context.Count);
-}
+        : new DateTime((long)context.Total / context.Count,DateTimeKind.Utc);
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/AvgIfAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/AvgIfAggregate.cs
@@ -70,5 +70,5 @@ internal static DateTime DtImpl(NumericAggregate context, DateTime n, bool t)
 
     internal static DateTime? DtImplFinish(NumericAggregate context) => context.Count == 0
         ? null
-        : new DateTime((long)context.Total / context.Count);
-}
+        : new DateTime((long)context.Total / context.Count,DateTimeKind.Utc);
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs
@@ -55,5 +55,5 @@ internal static DateTime DtImpl(NumericAggregate context, DateTime n)
     }
 
     internal static DateTime? DtImplFinish(NumericAggregate context)
-        => context.Count == 0 ? null : new DateTime((long)context.Total);
+        => context.Count == 0 ? null : new DateTime((long)context.Total,DateTimeKind.Utc);
 }
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxIfAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxIfAggregate.cs
@@ -65,5 +65,5 @@ internal static DateTime DtImpl(NumericAggregate context, TimeSpan n, bool t)
     }
 
     internal static DateTime? DtImplFinish(NumericAggregate context)
-        => context.Count == 0 ? null : new DateTime((long)context.Total);
-}
+        => context.Count == 0 ? null : new DateTime((long)context.Total, DateTimeKind.Utc);
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinAggregate.cs
@@ -55,5 +55,5 @@ internal static DateTime DtImpl(NumericAggregate context, DateTime n)
     }
 
     internal static DateTime? DtImplFinish(NumericAggregate context)
-        => context.Count == 0 ? null : new DateTime((long)context.Total);
+        => context.Count == 0 ? null : new DateTime((long)context.Total, DateTimeKind.Utc);
 }
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinIfAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinIfAggregate.cs
@@ -65,5 +65,5 @@ internal static DateTime DtImpl(NumericAggregate context, TimeSpan n, bool t)
     }
 
     internal static DateTime? DtImplFinish(NumericAggregate context)
-        => context.Count == 0 ? null : new DateTime((long)context.Total);
-}
+        => context.Count == 0 ? null : new DateTime((long)context.Total, DateTimeKind.Utc);
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs
@@ -123,6 +123,7 @@ static void AddCoalesce(List<ScalarOverloadInfo> overloads, Func<IScalarFunction
         TanFunction.Register(functions);
         SignFunction.Register(functions);
         RoundFunction.Register(functions);
+        ToHexFunction.Register(functions);
         RadiansFunction.Register(functions);
         DegreesFunction.Register(functions);
         LogFunction.Register(functions);

diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/CustomFunctions.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/CustomFunctions.cs
@@ -28,5 +28,6 @@ static CustomFunctions()
         DateTimeToIso.Register(functions);
         TrimWsFunction.Register(functions);
         ToDateTimeFmtFunction.Register(functions);
+        ParseHexFunction.Register(functions);
     }
-}
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ParseHexFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ParseHexFunction.cs
@@ -0,0 +1,19 @@
+using System;
+using System.Globalization;
+
+namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
+
+[KustoImplementation(Keyword = "parsehex")]
+internal partial class ParseHexFunction
+{
+    private static long? Impl(string s)
+    {
+        if (s.StartsWith("0x",StringComparison.InvariantCultureIgnoreCase))
+            s=s.Substring(2);
+
+        return long.TryParse(s, NumberStyles.AllowHexSpecifier,
+            CultureInfo.InvariantCulture, out var x)
+            ? x
+            : null;
+    }
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/RoundFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/RoundFunction.cs
@@ -7,6 +7,7 @@ namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
 [KustoImplementation(Keyword = "Functions.Round")]
 internal partial class RoundFunction
 {
-    private static double ImplP(double input, long precision) => Math.Round(input, (int)precision);
+    private static double DIImpl(double input, int precision) => Math.Round(input, precision);
+    private static double DLImpl(double input, long precision) => Math.Round(input, (int)precision);
     private static double Impl(double input) => Math.Round(input, 0);
 }
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ToHexFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ToHexFunction.cs
@@ -0,0 +1,10 @@
+using System;
+
+namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
+
+[KustoImplementation(Keyword = "Functions.ToHex")]
+internal partial class ToHexFunction
+{
+    private static string LongImpl(long input) => input.ToString("x");
+    private static string Impl(int input) => input.ToString("x");
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ToIntStringFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ToIntStringFunction.cs
@@ -1,4 +1,6 @@
-using System.Runtime.CompilerServices;
+using System.Globalization;
+using System;
+using System.Runtime.CompilerServices;
 
 
 namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
@@ -7,11 +9,22 @@ namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
 internal class ToIntStringFunction
 {
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static int? Impl(string input) =>
-        int.TryParse(input, out var parsedResult)
+    private static int? Impl(string input)
+    {
+        if (input.StartsWith("0x", StringComparison.InvariantCultureIgnoreCase))
+        {
+            input = input.Substring(2);
+
+            return int.TryParse(input, NumberStyles.AllowHexSpecifier,
+                CultureInfo.InvariantCulture, out var x)
+                ? x
+                : null;
+        }
+        return int.TryParse(input, out var parsedResult)
             ? parsedResult
             : (double.TryParse(input, out var parsedDouble) && !double.IsNaN(parsedDouble) &&
                !double.IsInfinity(parsedDouble))
                 ? (int)parsedDouble
                 : null;
-}
+    }
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ToLongStringFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/ToLongStringFunction.cs
@@ -1,4 +1,6 @@
-using System.Runtime.CompilerServices;
+using System.Globalization;
+using System;
+using System.Runtime.CompilerServices;
 
 
 namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
@@ -7,11 +9,23 @@ namespace KustoLoco.Core.Evaluation.BuiltIns.Impl;
 internal class ToLongStringFunction
 {
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static long? Impl(string input) =>
-        long.TryParse(input, out var parsedResult)
+    private static long? Impl(string input)
+    {
+        if (input.StartsWith("0x", StringComparison.InvariantCultureIgnoreCase))
+        {
+            input = input.Substring(2);
+
+            return long.TryParse(input, NumberStyles.AllowHexSpecifier,
+                CultureInfo.InvariantCulture, out var x)
+                ? x
+                : null;
+        }
+
+        return long.TryParse(input, out var parsedResult)
             ? parsedResult
             : (double.TryParse(input, out var parsedDouble) && !double.IsNaN(parsedDouble) &&
                !double.IsInfinity(parsedDouble))
                 ? (long)parsedDouble
                 : null;
-}
+    }
+}
diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/TrimWsFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/TrimWsFunction.cs
@@ -4,4 +4,4 @@
 internal partial class TrimWsFunction
 {
     private static string Impl(string s) => s.Trim();
-}
+}
diff --git a/libraries/KustoLoco.Core/Util/ColumnBuilder.cs b/libraries/KustoLoco.Core/Util/ColumnBuilder.cs
@@ -36,10 +36,18 @@ public override void Add(object? value)
     {
         if (typeof(T) == typeof(JsonNode))
             _data.Add((T?)value);
+        else
+        if (value is DateTime dt)
+        {
+            if (dt.Kind == DateTimeKind.Unspecified)
+                dt=DateTime.SpecifyKind(dt,DateTimeKind.Utc);
+            if (dt.Kind == DateTimeKind.Local) dt = dt.ToUniversalTime();
+            _data.Add(TypeMapping.CastOrConvertToNullable<T>(dt));
+        }
         else
             _data.Add(TypeMapping.CastOrConvertToNullable<T>(value));
     }
 
     public override BaseColumn ToColumn() => ColumnFactory.Create(_data.ToArray());
     public override Array GetDataAsArray() => _data.ToArray();
-}
+}
diff --git a/test/BasicTests/SimpleFunctionTests.cs b/test/BasicTests/SimpleFunctionTests.cs
@@ -542,4 +542,62 @@ public async Task QuoteSlash()
         result.Should().Contain("1");
     }
 
+
+    [TestMethod]
+    public async Task RoundDouble()
+    {
+        var query = "print round(3.14,1)";
+        var result = await LastLineOfResult(query);
+        result.Should().Be("3.1");
+    }
+
+
+
+    [TestMethod]
+    public async Task LogDouble()
+    {
+        var query = "print log(4)";
+        var result = await LastLineOfResult(query);
+        result.Should().StartWith("1.3");
+    }
+
+    [TestMethod]
+    public async Task SignInt()
+    {
+        var query = "print sign(-4)";
+        var result = await LastLineOfResult(query);
+        result.Should().Be("-1");
+    }
+
+    [TestMethod]
+    public async Task ToLongHex()
+    {
+        var query = "print parsehex('a0')";
+        var result = await LastLineOfResult(query);
+        result.Should().Be("160");
+    }
+
+    [TestMethod]
+    public async Task ToLongHexWithPrefix()
+    {
+        var query = "print parsehex('0xa0')";
+        var result = await LastLineOfResult(query);
+        result.Should().Be("160");
+    }
+
+    [TestMethod]
+    public async Task ToHex()
+    {
+        var query = "print tohex(160)";
+        var result = await LastLineOfResult(query);
+        result.Should().Be("a0");
+    }
+
+    [TestMethod]
+    public async Task ToLong2()
+    {
+        var query = "print tolong('0xa0')";
+        var result = await LastLineOfResult(query);
+        result.Should().Be("160");
+    }
 }
diff --git a/test/CoreTests/EndToEndTests.cs b/test/CoreTests/EndToEndTests.cs
@@ -1219,7 +1219,7 @@ public void BuiltIns_coalesce_Columnar()
         var expected = @"
 b:bool; i:int; l:long; r:real; dt:datetime; ts:timespan; s:string
 ------------------
-True; 1; 1; 1; 2023-01-01T00:00:00.0000000; 00:00:10; a
+True; 1; 1; 1; 2023-01-01T00:00:00.0000000Z; 00:00:10; a
 ";
 
         // Act & Assert
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,4 +4,4 @@ @@
     internal partial class TrimWsFunction
     {
         private static string Impl(string s) => s.Trim();
-    }
+    }