From a482a30df9f3a4c85f825c35de2b433a4206a07c Mon Sep 17 00:00:00 2001 From: Ahmad-Fuad Date: Fri, 31 May 2024 12:53:38 +0300 Subject: [PATCH] added missing references for sub-macros and renamed macros --- macros/generate_translated_table.sql | 12 ++++++------ macros/generate_translated_table_legacy.sql | 2 +- macros/group_names_mapping.sql | 4 ++-- macros/remove_group_name.sql | 8 ++++---- macros/remove_group_name_legacy.sql | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/macros/generate_translated_table.sql b/macros/generate_translated_table.sql index 51a5b2a..c55aa9a 100644 --- a/macros/generate_translated_table.sql +++ b/macros/generate_translated_table.sql @@ -3,13 +3,13 @@ {#-- If the macro is to be used to translate a repeat table model, then the remove group_names and repeat_table parameters need to be True --#} {#-- The langueage default is 'undefined' this can be changed by explicitly adding the language parameter when using the macro --#} -{#-- Example of minimum usage with referenced model as input, this utilizes set defaults: {{ generate_translated_table_v4('source', 'model', 'choice_table') }} --#} -{#-- Example of usage in a referenced model with specified language: {{ generate_translated_table_v4('source', 'model', 'choice_table', language='French (fr)') }} --#} -{#-- Example of usage in a source using remove group name parameter: {{ generate_translated_table_v4('source', 'table', remove_group_names=True) }} --#} -{#-- Example of usage in a referenced repeat table using the required parameters: {{ generate_translated_table_v4('source', 'model', 'choice_table', remove_group_names=True, repeat_table=True ) }} }} --#} +{#-- Example of minimum usage with referenced model as input, this utilizes set defaults: {{ generate_translated_table('source', 'model', 'choice_table') }} --#} +{#-- Example of usage in a referenced model with specified language: {{ generate_translated_table('source', 'model', 'choice_table', language='French (fr)') }} --#} +{#-- Example of usage in a source using remove group name parameter: {{ generate_translated_table('source', 'table', remove_group_names=True) }} --#} +{#-- Example of usage in a referenced repeat table using the required parameters: {{ generate_translated_table('source', 'model', 'choice_table', remove_group_names=True, repeat_table=True ) }} }} --#} -{% macro generate_translated_table_v4(source_name, sbm_table, labelstable=None, language=None, xfm_table=None, remove_group_names=False, repeat_table=False, exclude_columns=[]) %} +{% macro generate_translated_table(source_name, sbm_table, labelstable=None, language=None, xfm_table=None, remove_group_names=False, repeat_table=False, exclude_columns=[]) %} {%- set xfm_table = "xfm" ~ sbm_table[3:] if remove_group_names and not repeat_table else xfm_table -%} {%- set labelstable = "chc" ~ sbm_table[3:] if remove_group_names and not repeat_table else labelstable -%} @@ -18,7 +18,7 @@ {% if not remove_group_names %} {%- set fieldlist = dbt_utils.get_filtered_columns_in_relation(from=ref(sbm_table)) -%} {% else %} - {%- set mapping = group_names_mapping(source_name, sbm_table, xfm_table, repeat_table) -%} + {%- set mapping = ona_utils.group_names_mapping(source_name, sbm_table, xfm_table, repeat_table) -%} {%- set fieldlist = mapping['fieldlist'] %} {%- set fieldname_mapping = mapping['fieldname_mapping'] %} {% endif %} diff --git a/macros/generate_translated_table_legacy.sql b/macros/generate_translated_table_legacy.sql index 05a1767..abdfbe5 100644 --- a/macros/generate_translated_table_legacy.sql +++ b/macros/generate_translated_table_legacy.sql @@ -3,7 +3,7 @@ -- Example of minimum usage in a model, this utilizes defaults specified above: {{ generate_translated_table('airbyte', 'sbm_table_name'}} -- Example of usage in a model with specified language: {{ generate_translated_table('airbyte', 'sbm_table_name', language='French (fr)' }} -{% macro generate_translated_table(source_name, rawtable, labelstable=None, language=None, exclude_columns=[]) %} +{% macro generate_translated_table_legacy(source_name, rawtable, labelstable=None, language=None, exclude_columns=[]) %} {# Set default values for labelstable and language if not provided #} {%- set labelstable = "chc" ~ rawtable[3:] if labelstable is none else labelstable -%} diff --git a/macros/group_names_mapping.sql b/macros/group_names_mapping.sql index 044a877..788feed 100644 --- a/macros/group_names_mapping.sql +++ b/macros/group_names_mapping.sql @@ -1,7 +1,7 @@ {% macro group_names_mapping(source_name, sbm_table, xfm_table, repeat_table) %} {% if not repeat_table %} {%- set colnames = dbt_utils.get_filtered_columns_in_relation(from=source(source_name, sbm_table)) -%} - {%- set xfm_dict = group_field_dictionary(source_name, xfm_table) -%} + {%- set xfm_dict = ona_utils.group_field_dictionary(source_name, xfm_table) -%} {% else %} {%- set colnames = dbt_utils.get_filtered_columns_in_relation(from=ref(sbm_table)) -%} {% endif %} @@ -12,7 +12,7 @@ {% for column in colnames %} {% set fieldname = column %} {%- if not repeat_table and column not in exclude_columns %} - {% set fieldname = find_index(xfm_dict['airbyte_name'], column, xfm_dict['field']) %} + {% set fieldname = ona_utils.find_index(xfm_dict['airbyte_name'], column, xfm_dict['field']) %} {% elif repeat_table and column not in exclude_columns and '/' in column %} {% set fieldname = fieldname[fieldname.rfind('/')+1:] %} {% endif %} diff --git a/macros/remove_group_name.sql b/macros/remove_group_name.sql index 894789b..8e68c0e 100644 --- a/macros/remove_group_name.sql +++ b/macros/remove_group_name.sql @@ -1,14 +1,14 @@ -- Macro to remove group names from data pushed by Airbyte Ona Data connector to Destination V2 -- This macro can be used to refrence a source or model(in case of repeat tables) in dbt to rename all columns recursively, also provides a option for column exclusion -- Sample usage with source as input: {{ remove_group_name('source', 'table', ['group1/column1', 'group2/column1']) }} --# --- Sample usage with repeat table as input: {{remove_group_names_v2(sbm_table='repeat_model_name', repeat_table=True, ['group1/column1', 'group2/column1']) }} +-- Sample usage with repeat table as input: {{remove_group_name(sbm_table='repeat_model_name', repeat_table=True, ['group1/column1', 'group2/column1']) }} -{% macro remove_group_names_v2(source_name, sbm_table, xfm_table=None, repeat_table=False, exclude_columns=[]) %} +{% macro remove_group_name(source_name, sbm_table, xfm_table=None, repeat_table=False, exclude_columns=[]) %} {% if not repeat_table %} {%- set xfm_table = "xfm" ~ sbm_table[3:] if xfm_table is none else xfm_table -%} {%- set fieldlist = dbt_utils.get_filtered_columns_in_relation(from=source(source_name, sbm_table)) -%} - {%- set xfm_dict = group_field_dictionary(source_name, xfm_table) -%} + {%- set xfm_dict = ona_utils.group_field_dictionary(source_name, xfm_table) -%} {% else %} {%- set fieldlist = dbt_utils.get_filtered_columns_in_relation(from=ref(sbm_table)) -%} {% endif %} @@ -16,7 +16,7 @@ select {% for field in fieldlist %} {%- if not repeat_table and field not in exclude_columns %} - "{{ field }}" as "{{ find_index(xfm_dict['airbyte_name'], field, xfm_dict['field']) }}" + "{{ field }}" as "{{ ona_utils.find_index(xfm_dict['airbyte_name'], field, xfm_dict['field']) }}" {%- elif repeat_table and field not in exclude_columns and '/' in field %} "{{ field }}" as {{ field[field.rfind('/')+1:] }} {%- else -%} diff --git a/macros/remove_group_name_legacy.sql b/macros/remove_group_name_legacy.sql index 7520a0e..66cbb2e 100644 --- a/macros/remove_group_name_legacy.sql +++ b/macros/remove_group_name_legacy.sql @@ -2,7 +2,7 @@ {#-- Sample usage: in a staging model, {{ remove_group_name('source', 'table', ['group1/column1', 'group2/column1']) }} --#} -{% macro remove_group_name(source_name, table_name, exclude_columns=[]) %} +{% macro remove_group_name_legacy(source_name, table_name, exclude_columns=[]) %} {% set colnames = dbt_utils.get_filtered_columns_in_relation(from=source(source_name, table_name)) %} select