Skip to content

Commit

Permalink
Merge pull request #29 from data-solution-automation-engine/dev-2024-06
Browse files Browse the repository at this point in the history
Dev 2024 06 - 2.1.0 version
  • Loading branch information
RoelantVos authored Jul 8, 2024
2 parents 80dfe2d + 537e499 commit 17121fc
Show file tree
Hide file tree
Showing 35 changed files with 837 additions and 1,011 deletions.
33 changes: 11 additions & 22 deletions DataWarehouseAutomation/DataWarehouseAutomation.sln
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
Images\Ravos-logo.png = Images\Ravos-logo.png
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Validation", "Test_Project\Validation.csproj", "{23CF4A46-B17F-4642-B949-02E061868BBC}"
ProjectSection(ProjectDependencies) = postProject
{F0458983-9646-42E6-9E18-90AD91288B40} = {F0458983-9646-42E6-9E18-90AD91288B40}
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample Metadata", "Sample Metadata", "{242F4BD0-68F9-4193-A329-826E755C6B08}"
ProjectSection(SolutionItems) = preProject
Sample_Metadata\myFirstMapping.json = Sample_Metadata\myFirstMapping.json
Sample_Metadata\sampleBasic.json = Sample_Metadata\sampleBasic.json
Sample_Metadata\sampleBasicWithExtensions.json = Sample_Metadata\sampleBasicWithExtensions.json
Sample_Metadata\sampleCalculation.json = Sample_Metadata\sampleCalculation.json
Sample_Metadata\sampleCustomFunctions.json = Sample_Metadata\sampleCustomFunctions.json
Sample_Metadata\sampleDataVaultHub.json = Sample_Metadata\sampleDataVaultHub.json
Sample_Metadata\sampleFreeForm.json = Sample_Metadata\sampleFreeForm.json
Sample_Metadata\sampleJsonStagingWithPsaDetails.json = Sample_Metadata\sampleJsonStagingWithPsaDetails.json
Sample_Metadata\sampleMultipleDataItemMappings.json = Sample_Metadata\sampleMultipleDataItemMappings.json
Sample_Metadata\sampleSimpleDDL.json = Sample_Metadata\sampleSimpleDDL.json
Sample_Metadata\sampleSourceQuery.json = Sample_Metadata\sampleSourceQuery.json
Expand All @@ -46,13 +39,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample Templates", "Sample
Sample_Templates\TemplateSampleSourceQuery.Handlebars = Sample_Templates\TemplateSampleSourceQuery.Handlebars
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Examples", "Example_Project\Examples.csproj", "{F77AC518-6300-4927-8775-B92CDF15CC8E}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RunDwhAutomation", "RunDwhAutomation\RunDwhAutomation.csproj", "{62C992C6-370D-4591-B54E-2F61E2D8B638}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SchemaJsonConverter", "SchemaJsonConverter\SchemaJsonConverter.csproj", "{07214B80-37A9-4137-B0F3-109FA0D0926A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Export", "Export\Export.csproj", "{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Validation", "Validation\Validation.csproj", "{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Examples", "Examples\Examples.csproj", "{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand All @@ -64,14 +57,6 @@ Global
{F0458983-9646-42E6-9E18-90AD91288B40}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F0458983-9646-42E6-9E18-90AD91288B40}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F0458983-9646-42E6-9E18-90AD91288B40}.Release|Any CPU.Build.0 = Release|Any CPU
{23CF4A46-B17F-4642-B949-02E061868BBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{23CF4A46-B17F-4642-B949-02E061868BBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{23CF4A46-B17F-4642-B949-02E061868BBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{23CF4A46-B17F-4642-B949-02E061868BBC}.Release|Any CPU.Build.0 = Release|Any CPU
{F77AC518-6300-4927-8775-B92CDF15CC8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F77AC518-6300-4927-8775-B92CDF15CC8E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F77AC518-6300-4927-8775-B92CDF15CC8E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F77AC518-6300-4927-8775-B92CDF15CC8E}.Release|Any CPU.Build.0 = Release|Any CPU
{62C992C6-370D-4591-B54E-2F61E2D8B638}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{62C992C6-370D-4591-B54E-2F61E2D8B638}.Debug|Any CPU.Build.0 = Debug|Any CPU
{62C992C6-370D-4591-B54E-2F61E2D8B638}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand All @@ -80,10 +65,14 @@ Global
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.Build.0 = Release|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.Build.0 = Release|Any CPU
{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DB4DA80B-39D0-451F-ADF4-C4CA75F1851D}.Release|Any CPU.Build.0 = Release|Any CPU
{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{59169ABF-D3E5-479E-B4F5-D30DEB5900A2}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<Version>2.0.3</Version>
<Version>2.1.0</Version>
<Authors>Roelant Vos</Authors>
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
<Title>Data Warehouse Automation</Title>
<PackageProjectUrl>https://roelantvos.com/blog/interface-for-data-warehouse-automation/</PackageProjectUrl>
<PackageProjectUrl>https://github.com/data-solution-automation-engine/data-warehouse-automation-metadata-schema</PackageProjectUrl>
<PackageIcon>Ravos-logo.png</PackageIcon>
<PackageReadmeFile>index.md</PackageReadmeFile>
<RepositoryUrl>https://github.com/RoelantVos/data-warehouse-automation-metadata-schema</RepositoryUrl>
<RepositoryUrl>https://github.com/data-solution-automation-engine/data-warehouse-automation-metadata-schema</RepositoryUrl>
<PackageTags>data warehouse; etl; data integration; data warehouse automation</PackageTags>
<Description>The generic schema for Data Warehouse Automation covers the efforts towards defining a standard interface to exchange Data Warehouse Automation and ETL generation metadata.

Expand All @@ -35,6 +35,22 @@ The interface itself is a Json schema definition (JSDL) with examples on how to
<PackageReference Include="Handlebars.Net" Version="2.1.4" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Newtonsoft.Json.Schema" Version="3.0.15" />
<PackageReference Include="Roslynator.Analyzers" Version="4.12.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.CodeFixes" Version="4.12.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.12.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.Refactorings" Version="4.12.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
namespace DataWarehouseAutomation.DwaModel;

/// <summary>
/// A Business Key, which consists of one or more components (column mappings) and has its own surrogate key.
/// A Business Key is a special column, or combination of columns, that is defined separately outside of regular data item mappings.
/// A business key definition is a special object that is defined as an optional property of a data object mapping.
/// In other words, the business key definition can be a part of describing the relationship between a source data object and a target data object.
/// </summary>
public class BusinessKeyDefinition
{
Expand All @@ -14,16 +14,18 @@ public class BusinessKeyDefinition
public string? Id { get; set; }

/// <summary>
/// The optional name of the Business Key Definition.
/// The optional name of the business key definition.
/// </summary>
[JsonPropertyName("name")]
public string? Name { get; set; }

/// <summary>
/// Items that define the Business Key e.g. the collection of columns for a Business Key.
/// Items that define the Business Key e.g. the collection of columns for a business key.
/// The integer value defines the order of the components in the business key definition.
/// The order is significant, and is recorded as part of the tuple definition.
/// </summary>
[JsonPropertyName("businessKeyComponentMappings")]
public List<DataItemMapping> BusinessKeyComponentMappings { get; set; } = new();
[JsonPropertyName("businessKeyComponents")]
public List<BusinessKeyComponents>? BusinessKeyComponents { get; set; }

/// <summary>
/// An optional label for the end result e.g. the target business key attribute.
Expand All @@ -33,14 +35,14 @@ public class BusinessKeyDefinition
public string? SurrogateKey { get; set; }

/// <summary>
/// Free-form and optional classification for the Business Key for use in generation logic (evaluation).
/// Free-form and optional classification for the business key for use in generation logic (evaluation).
/// </summary>
[JsonPropertyName("classifications")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public List<DataClassification>? Classifications { get; set; }

/// <summary>
/// The collection of extension Key/Value pairs.
/// The collection of extension key/value pairs.
/// </summary>
[JsonPropertyName("extensions")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
namespace DataWarehouseAutomation.DwaModel;

/// <summary>
/// This object captures the cardinality and ordinality of a relationship.
/// Cardinality refers to the uniqueness of data values contained in a column (attribute) of a database table.
/// It defines the number of occurrences of one entity that are associated with the number of occurrences of another entity through a relationship.
/// </summary>
public class Cardinality
{
/// <summary>
/// Optional identifier as a string value to allow various identifier approaches.
/// </summary>
[JsonPropertyName("id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string? Id { get; set; }

/// <summary>
/// Optional information to name a certain cardinality construct.
/// For example one-to-one, one-to-many, or many-to-many.
/// E.g. one-to-one could be defined as {"fromRange": {"min": "1", "max": "1"}, "toRange": {"min": "1", "max": "1"}}.
/// </summary>
[JsonPropertyName("name")]
public string? Name { get; set; }

/// <summary>
/// The 'from' component in the cardinality, e.g. the '1' in 1 to many.
/// </summary>
[JsonPropertyName("fromRange")]
public CardinalityRange? FromRange { get; set; } = new CardinalityRange { Min = "1", Max = "1" };

/// <summary>
/// The 'to' component in the cardinality, e.g. the 'many' in 1 to many.
/// </summary>
[JsonPropertyName("toRange")]
public CardinalityRange? ToRange { get; set; } = new CardinalityRange { Min = "1", Max = "N" };

/// <summary>
/// Free-form and optional classification for the Cardinality for use in generation logic (evaluation).
/// </summary>
[JsonPropertyName("classifications")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public List<DataClassification>? Classifications { get; set; }

/// <summary>
/// The collection of extension Key/Value pairs.
/// </summary>
[JsonPropertyName("extensions")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public List<Extension>? Extensions { get; set; }

/// <summary>
/// Free-format notes.
/// </summary>
[JsonPropertyName("notes")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string? Notes { get; set; }

#region Methods
/// <summary>
/// Use this method to assert if two Cardinalities are the same, based on their Ids.
/// </summary>
/// <param name="obj"></param>
/// <returns>True if the Cardinalities are the same, based on their Ids</returns>
public override bool Equals(object? obj)
{
var other = obj as Cardinality;
return other?.Id == Id;
}

/// <summary>
/// Override to get a hash value that represents the identifier.
/// </summary>
/// <returns>A 32-bit signed integer hash code</returns>
public override int GetHashCode() => (Id?.GetHashCode()) ?? 0;

/// <summary>
/// String override so that the object returns its value ('Name').
/// When an instance of this class is passed to a method that expects a string, the ToString() method will be called implicitly to convert the object to a string, and the value of the "Name" property will be returned.
/// </summary>
/// <returns>The Name</returns>
public override string ToString()
{
return Name ?? string.Empty;
}
#endregion
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
namespace DataWarehouseAutomation.DwaModel;

/// <summary>
/// The possible range for a from/to component for the cardinality.
/// For example "min": "1", "max": "N"
/// This way, you can define "at least 1 to many". Or "0 or 1 to 1".
/// </summary>
public class CardinalityRange
{
[JsonPropertyName("min")]
public string? Min { get; set; }

[JsonPropertyName("max")]
public string? Max { get; set; }
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
namespace DataWarehouseAutomation.DwaModel;

/// <summary>
/// Data items belong to data objects or data object queries. This means that a given data item always has a parent object.
/// They describe the individual elements, such as the columns in a table or headers in a file.
/// </summary>
public class DataItem : IDataItem
{
/// <summary>
Expand All @@ -14,7 +18,7 @@ public class DataItem : IDataItem
public string Name { get; set; } = "NewDataItem";

/// <summary>
/// The data object to which the data item belongs. This can be used to construct fully qualified names.
/// The <see cref="IDataObject"/> to which the data item belongs. This can be used to construct fully qualified names.
/// </summary>
[JsonPropertyName("dataObject")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
namespace DataWarehouseAutomation.DwaModel;

/// <summary>
/// The individual column-to-column mapping.
/// The individual <see cref="IDataItem"/> to <see cref="IDataItem"/>, column-to-column mapping.
/// </summary>
public class DataItemMapping
{
Expand All @@ -13,7 +13,7 @@ public class DataItemMapping
public string? Id { get; set; }

[JsonPropertyName("sourceDataItems")]
public List<IDataItem> SourceDataItems { get; set; } = new();
public List<IDataItem> SourceDataItems { get; set; } = [];

[JsonPropertyName("targetDataItem")]
public IDataItem TargetDataItem { get; set; } = new DataItem() { Name = "newTargetDataItem" };
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ public class DataItemQuery : IDataItem
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public string Name { get; set; } = string.Empty;

// convenience property for parent, derived on json load, never stored in the json file
// Can be either a DataObject or a DataQuery when the DataQuery is a DataItem level thing
/// <summary>
/// The <see cref="IDataObject"/> to which the data item belongs. This can be used to construct fully qualified names.
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public IDataObject? DataObject { get; set; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@

/// <summary>
/// The definition of a data set, file, or table.
/// The Data Object can be the 'source' or 'target' in a Data Object Mapping.
/// A Data Object which acts as target in one mapping, can be a source in another mapping, building up the data logistics lineage.
/// The Data Object can be the 'source' or 'target' in a <see cref="DataObjectMapping"/>.
/// A Data Object which acts as target in one mapping, can be a source in another mapping,
/// building up the data logistics lineage.
/// </summary>
public class DataObject : IMetadata, IDataObject
{
Expand Down Expand Up @@ -35,7 +36,24 @@ public class DataObject : IMetadata, IDataObject
public DataConnection? DataConnection { get; set; }

/// <summary>
/// Free-form and optional classification for the Data Object for use in ETL generation logic (evaluation).
/// The definition of the Business Key(s) for the Data Object.
/// Being able to record the business key definition
/// This serves multiple purposes, but one of them is to support defining a series of business key definitions against the source data object, and reuse these across different data object mappings.
/// The order is stored as well, because in some cases the order of keys is meaningful.
/// </summary>
[JsonPropertyName("businessKeyDefinitions")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public List<BusinessKeyDefinitions>? BusinessKeyDefinitions { get; set; }

/// <summary>
/// Any relationship to other data objects.
/// </summary>
[JsonPropertyName("relationships")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
public List<Relationships>? Relationships { get; set; }

/// <summary>
/// Free-form and optional classification for the Data Object.
/// </summary>
[JsonPropertyName("classifications")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
Expand Down
Loading

0 comments on commit 17121fc

Please sign in to comment.