Merge branch 'CoreHelpers:master' into feature/onlyrowandpartition

CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests/CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0"/>
+        <PackageReference Include="xunit" Version="2.4.1"/>
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+        <PackageReference Include="coverlet.collector" Version="3.1.2">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\CoreHelpers.WindowsAzure.Storage.Table.Backup\CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj" />
+    </ItemGroup>
+
+</Project>

CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests/TableExcludeExpressionTests.cs

@@ -0,0 +1,31 @@
+using CoreHelpers.WindowsAzure.Storage.Table.Backup.Internal;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests;
+
+public class TableExcludeExpressionTests
+{
+    [Fact]
+    public void VerifyFullTableNameTests()
+    {
+        var validator = new TabledExcludeExpressionValidator(new string[] { "demoTable", "secondTable" });
+        Assert.True(validator.IsTableExcluded("demoTable"));
+        Assert.True(validator.IsTableExcluded("demotable"));
+        Assert.True(validator.IsTableExcluded("DEMOTABLE"));
+        Assert.True(validator.IsTableExcluded("secondtable"));
+        Assert.False(validator.IsTableExcluded("demo*"));
+        Assert.False(validator.IsTableExcluded("*Table"));
+        Assert.False(validator.IsTableExcluded("otherTable"));
+    }
+
+    [Fact]
+    public void VerifyRegExPatternTableNameTests()
+    {
+        var validator = new TabledExcludeExpressionValidator(new string[] { "demo.*" });
+        Assert.True(validator.IsTableExcluded("demoTable"));
+        Assert.True(validator.IsTableExcluded("demotable"));
+        Assert.True(validator.IsTableExcluded("DEMOTABLE"));
+        Assert.True(validator.IsTableExcluded("demo*"));
+        Assert.False(validator.IsTableExcluded("*Table"));
+        Assert.False(validator.IsTableExcluded("otherTable"));
+    }
+}

CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests/Usings.cs

@@ -0,0 +1 @@
+global using Xunit;

CoreHelpers.WindowsAzure.Storage.Table.Backup/AssemblyInfo.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+[assembly: AssemblyCompany("Core Helpers")]
+[assembly: AssemblyProduct("WindowsAzure.Storage.Table.Backup")]
+[assembly: AssemblyTitle("CoreHelpers Azure Storage Abstraction")]
+
+[assembly: AssemblyFileVersion("2.0.0.0")]
+[assembly: AssemblyVersion("2.0.0.0")]
+
+#if (DEBUG)
+[assembly: AssemblyConfiguration("Debug")]
+#else
+[assembly: AssemblyConfiguration("Release")]
+#endif
+
+[assembly: InternalsVisibleTo("CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests")]

CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs

@@ -5,6 +5,7 @@
 using System.Threading.Tasks;
 using Azure.Storage.Blobs;
 using CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions;
+using CoreHelpers.WindowsAzure.Storage.Table.Backup.Internal;
 using Microsoft.Extensions.Logging;
 
 namespace CoreHelpers.WindowsAzure.Storage.Table.Backup
@@ -34,15 +35,9 @@ public async Task Backup(IStorageContext storageContext, string[] excludedTables
         {
             using (_logger.BeginScope("Starting backup procedure..."))
             {
-
-                // generate the excludeTables
-                var excludedTablesList = new List<string>();
-                if (excludedTables != null)
-                {
-                    foreach (var tbl in excludedTables)
-                        excludedTablesList.Add(tbl.ToLower());
-                }
-
+                // build the exclude validator
+                var excludeValidator = new TabledExcludeExpressionValidator(excludedTables);
+
                 // get all tables 
                 var tables = await storageContext.QueryTableList();
                 _logger.LogInformation($"Processing {tables.Count} tables");
@@ -78,7 +73,7 @@ public async Task Backup(IStorageContext storageContext, string[] excludedTables
                         }
 
                         // check the  excluded tables
-                        if (excludedTablesList.Contains(tableName.ToLower()))
+                        if (excludeValidator.IsTableExcluded(tableName))
                         {
                             _logger.LogInformation($"Ignoring table {tableName} (is part of excluded tables)...");
                             continue;

CoreHelpers.WindowsAzure.Storage.Table.Backup/Internal/TabledExcludeExpressionValidator.cs

@@ -0,0 +1,32 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup.Internal;
+
+internal class TabledExcludeExpressionValidator
+{
+    private List<string> _excludes = new List<string>();
+    public TabledExcludeExpressionValidator(string[] excludes)
+    {
+        if (excludes != null)
+            _excludes = excludes.Select(e => e.ToLower()).ToList();
+    }
+
+    public bool IsTableExcluded(string tableName)
+    {
+        // verify if the tablename is in the exclude list
+        if (_excludes.Contains(tableName.ToLower()))
+            return true;
+
+        // verify if the tablename matches a regex pattern
+        foreach (var exclude in _excludes)
+        { 
+            if (Regex.IsMatch(tableName, exclude, RegexOptions.IgnoreCase)) 
+                return true;
+        }
+
+        // all the rest don't exclude
+        return false;
+    } 
+}

CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS016ExportToJson.cs

@@ -1,4 +1,4 @@
-using System.Text;
+using System.Text;
 using CoreHelpers.WindowsAzure.Storage.Table.Tests.Contracts;
 using CoreHelpers.WindowsAzure.Storage.Table.Tests.Extensions;
 using CoreHelpers.WindowsAzure.Storage.Table.Tests.Models;
@@ -32,7 +32,7 @@ public async Task VerifyExportToJson()
                 storageContext.AddAttributeMapper(typeof(DemoModel2), tableName1);
 
                 // create model with data in list
-                var model = new DemoModel2() { P = "1", R = "2" };
+                var model = new DemoModel2() { P = "1", R = "2", CreatedAt = DateTime.Parse("2023-11-10T19:09:50.065462+00:00").ToUniversalTime()};
 
                 // inser the model                    
                 await storageContext.EnableAutoCreateTable().MergeOrInsertAsync<DemoModel2>(new List<DemoModel2>() { model });
@@ -46,7 +46,7 @@ public async Task VerifyExportToJson()
 
                 // verify the targetstream
                 var parsedStream = Encoding.Default.GetString(targetStream.GetBuffer()).Split("\0")[0];
-                var expectedStreamValue = "[{\"RowKey\":\"2\",\"PartitionKey\":\"1\",\"Properties\":[{\"PropertyName\":\"P\",\"PropertyType\":0,\"PropertyValue\":\"1\"},{\"PropertyName\":\"R\",\"PropertyType\":0,\"PropertyValue\":\"2\"}]}]";
+                var expectedStreamValue = "[{\"RowKey\":\"2\",\"PartitionKey\":\"1\",\"Properties\":[{\"PropertyName\":\"CreatedAt\",\"PropertyType\":3,\"PropertyValue\":\"2023-11-10T19:09:50.065462+00:00\"},{\"PropertyName\":\"P\",\"PropertyType\":0,\"PropertyValue\":\"1\"},{\"PropertyName\":\"R\",\"PropertyType\":0,\"PropertyValue\":\"2\"}]}]";
                 Assert.Equal(expectedStreamValue, parsedStream);
 
                 // drop table

CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS017ImportFromJson.cs

@@ -1,4 +1,4 @@
-using System.Text;
+using System.Text;
 using CoreHelpers.WindowsAzure.Storage.Table.Tests.Contracts;
 using CoreHelpers.WindowsAzure.Storage.Table.Tests.Extensions;
 using CoreHelpers.WindowsAzure.Storage.Table.Tests.Models;
@@ -34,7 +34,7 @@ public async Task VerifyImportFromJson()
                 storageContext.AddAttributeMapper(typeof(DemoModel2), tableName1);
 
                 // define the import data 
-                var staticExportData = "[{\"RowKey\":\"2\",\"PartitionKey\":\"1\",\"Properties\":[{\"PropertyName\":\"P\",\"PropertyType\":0,\"PropertyValue\":\"1\"},{\"PropertyName\":\"R\",\"PropertyType\":0,\"PropertyValue\":\"2\"}]}]";
+                var staticExportData = "[{\"RowKey\":\"2\",\"PartitionKey\":\"1\",\"Properties\":[{\"PropertyName\":\"P\",\"PropertyType\":0,\"PropertyValue\":\"1\"},{\"PropertyName\":\"R\",\"PropertyType\":0,\"PropertyValue\":\"2\"},{\"PropertyName\":\"CreatedAt\",\"PropertyType\":3,\"PropertyValue\":\"2023-01-30T22:58:40.5859427+00:00\"}]}]";
                 var staticExportDataStream = new MemoryStream(Encoding.UTF8.GetBytes(staticExportData ?? ""));
 
                 // check if we have an empty tabel before import
@@ -53,7 +53,13 @@ public async Task VerifyImportFromJson()
                 // get the data
                 var data = await storageContext.Query<DemoModel2>().Now();
                 Assert.Equal("1", data.First().P);
-                Assert.Equal("2", data.First().R);                
+                Assert.Equal("2", data.First().R);
+
+                var createdAtDate = DateTime.Parse("2023-01-30T22:58:40.5859427+00:00");
+                var createdAtDateFromDataLoad = data.First().CreatedAt;
+
+                Assert.Equal(createdAtDate.ToUniversalTime(), createdAtDateFromDataLoad);
+                Assert.Equal(createdAtDate.ToUniversalTime(), createdAtDateFromDataLoad.ToUniversalTime());
 
                 // drop table
                 await storageContext.DropTableAsync<DemoModel2>();                

CoreHelpers.WindowsAzure.Storage.Table.Tests/Models/DemoModel2.cs

@@ -12,6 +12,8 @@ public class DemoModel2
 
         [RowKey]
         public string R { get; set; } = "R1";
+
+        public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
     }
 }
 

CoreHelpers.WindowsAzure.Storage.Table.sln

@@ -18,6 +18,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CoreHelpers.WindowsAzure.St
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions", "CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions\CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions.csproj", "{72CBB151-FEEC-4174-AEAE-684EAAF951AB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests", "CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests\CoreHelpers.WindowsAzure.Storage.Table.Backup.Tests.csproj", "{E2137490-9013-41E8-8CF3-F814414E7FD8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -44,6 +46,10 @@ Global
 		{72CBB151-FEEC-4174-AEAE-684EAAF951AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{72CBB151-FEEC-4174-AEAE-684EAAF951AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{72CBB151-FEEC-4174-AEAE-684EAAF951AB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E2137490-9013-41E8-8CF3-F814414E7FD8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E2137490-9013-41E8-8CF3-F814414E7FD8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E2137490-9013-41E8-8CF3-F814414E7FD8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E2137490-9013-41E8-8CF3-F814414E7FD8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

CoreHelpers.WindowsAzure.Storage.Table/StorageContextImportExport.cs

@@ -27,7 +27,7 @@ public async Task ExportToJsonAsync(string tableName, TextWriter writer, Action<
 
                 // build the json writer
                 JsonWriter wr = new JsonTextWriter(writer);
-
+                
                 // prepare the array in result
                 wr.WriteStartArray();
 
@@ -63,7 +63,17 @@ public async Task ExportToJsonAsync(string tableName, TextWriter writer, Action<
                             wr.WritePropertyName(TableConstants.PropertyType);
                             wr.WriteValue(propertyKvp.Value.GetType().GetEdmPropertyType());
                             wr.WritePropertyName(TableConstants.PropertyValue);
-                            wr.WriteValue(propertyKvp.Value);
+
+                            switch (propertyKvp.Value.GetType().GetEdmPropertyType())
+                            {
+                                case ExportEdmType.DateTime:
+                                    wr.WriteValue(((DateTimeOffset)propertyKvp.Value).ToUniversalTime());
+                                    break;
+                                default:
+                                    wr.WriteValue(propertyKvp.Value);
+                                    break;
+                            }
+
                             wr.WriteEndObject();
                         }
                         wr.WriteEnd();
@@ -113,6 +123,8 @@ public async Task ImportFromJsonAsync(string tableName, StreamReader reader, Act
                             if ((ExportEdmType)property.PropertyType == ExportEdmType.String && property.PropertyValue is DateTime)
                             {
                                 property.PropertyValue = ((DateTime)property.PropertyValue).ToString("o");
+                            } else if ((ExportEdmType)property.PropertyType == ExportEdmType.DateTime) {
+                                property.PropertyValue = ((DateTime)property.PropertyValue).ToUniversalTime();
                             }
                         }